{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<script>requirejs.config({paths: { 'plotly': ['https://cdn.plot.ly/plotly-latest.min']},});if(!window.Plotly) {{require(['plotly'],function(plotly) {window.Plotly=plotly;});}}</script>"
      ],
      "text/vnd.plotly.v1+html": [
       "<script>requirejs.config({paths: { 'plotly': ['https://cdn.plot.ly/plotly-latest.min']},});if(!window.Plotly) {{require(['plotly'],function(plotly) {window.Plotly=plotly;});}}</script>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Python version: 3.6.3 |Anaconda custom (64-bit)| (default, Oct 15 2017, 03:27:45) [MSC v.1900 64 bit (AMD64)]\n",
      "pandas version: 0.20.3\n",
      "NumPy version: 1.14.3\n",
      "SciPy version: 1.1.0\n",
      "scikit-learn version: 0.19.1\n",
      "matplotlib version: 2.1.0\n",
      "IPython version: 6.1.0\n",
      "Using Hard-Coded Configuration for data_file_path\n",
      "data_file_path:  C:/Development/kaggle--home-credit-default-risk/data/\n",
      "data_file_name:  application_train.csv\n",
      "\n",
      " df_row_count:  307511 \n",
      " df_column_count:  122 \n",
      "\n",
      "\n",
      " df_values_count_total:  37516342 \n",
      "\n",
      "\n",
      " df_column_names:  \n",
      " ['SK_ID_CURR', 'TARGET', 'NAME_CONTRACT_TYPE', 'CODE_GENDER', 'FLAG_OWN_CAR', 'FLAG_OWN_REALTY', 'CNT_CHILDREN', 'AMT_INCOME_TOTAL', 'AMT_CREDIT', 'AMT_ANNUITY', 'AMT_GOODS_PRICE', 'NAME_TYPE_SUITE', 'NAME_INCOME_TYPE', 'NAME_EDUCATION_TYPE', 'NAME_FAMILY_STATUS', 'NAME_HOUSING_TYPE', 'REGION_POPULATION_RELATIVE', 'DAYS_BIRTH', 'DAYS_EMPLOYED', 'DAYS_REGISTRATION', 'DAYS_ID_PUBLISH', 'OWN_CAR_AGE', 'FLAG_MOBIL', 'FLAG_EMP_PHONE', 'FLAG_WORK_PHONE', 'FLAG_CONT_MOBILE', 'FLAG_PHONE', 'FLAG_EMAIL', 'OCCUPATION_TYPE', 'CNT_FAM_MEMBERS', 'REGION_RATING_CLIENT', 'REGION_RATING_CLIENT_W_CITY', 'WEEKDAY_APPR_PROCESS_START', 'HOUR_APPR_PROCESS_START', 'REG_REGION_NOT_LIVE_REGION', 'REG_REGION_NOT_WORK_REGION', 'LIVE_REGION_NOT_WORK_REGION', 'REG_CITY_NOT_LIVE_CITY', 'REG_CITY_NOT_WORK_CITY', 'LIVE_CITY_NOT_WORK_CITY', 'ORGANIZATION_TYPE', 'EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3', 'APARTMENTS_AVG', 'BASEMENTAREA_AVG', 'YEARS_BEGINEXPLUATATION_AVG', 'YEARS_BUILD_AVG', 'COMMONAREA_AVG', 'ELEVATORS_AVG', 'ENTRANCES_AVG', 'FLOORSMAX_AVG', 'FLOORSMIN_AVG', 'LANDAREA_AVG', 'LIVINGAPARTMENTS_AVG', 'LIVINGAREA_AVG', 'NONLIVINGAPARTMENTS_AVG', 'NONLIVINGAREA_AVG', 'APARTMENTS_MODE', 'BASEMENTAREA_MODE', 'YEARS_BEGINEXPLUATATION_MODE', 'YEARS_BUILD_MODE', 'COMMONAREA_MODE', 'ELEVATORS_MODE', 'ENTRANCES_MODE', 'FLOORSMAX_MODE', 'FLOORSMIN_MODE', 'LANDAREA_MODE', 'LIVINGAPARTMENTS_MODE', 'LIVINGAREA_MODE', 'NONLIVINGAPARTMENTS_MODE', 'NONLIVINGAREA_MODE', 'APARTMENTS_MEDI', 'BASEMENTAREA_MEDI', 'YEARS_BEGINEXPLUATATION_MEDI', 'YEARS_BUILD_MEDI', 'COMMONAREA_MEDI', 'ELEVATORS_MEDI', 'ENTRANCES_MEDI', 'FLOORSMAX_MEDI', 'FLOORSMIN_MEDI', 'LANDAREA_MEDI', 'LIVINGAPARTMENTS_MEDI', 'LIVINGAREA_MEDI', 'NONLIVINGAPARTMENTS_MEDI', 'NONLIVINGAREA_MEDI', 'FONDKAPREMONT_MODE', 'HOUSETYPE_MODE', 'TOTALAREA_MODE', 'WALLSMATERIAL_MODE', 'EMERGENCYSTATE_MODE', 'OBS_30_CNT_SOCIAL_CIRCLE', 'DEF_30_CNT_SOCIAL_CIRCLE', 'OBS_60_CNT_SOCIAL_CIRCLE', 'DEF_60_CNT_SOCIAL_CIRCLE', 'DAYS_LAST_PHONE_CHANGE', 'FLAG_DOCUMENT_2', 'FLAG_DOCUMENT_3', 'FLAG_DOCUMENT_4', 'FLAG_DOCUMENT_5', 'FLAG_DOCUMENT_6', 'FLAG_DOCUMENT_7', 'FLAG_DOCUMENT_8', 'FLAG_DOCUMENT_9', 'FLAG_DOCUMENT_10', 'FLAG_DOCUMENT_11', 'FLAG_DOCUMENT_12', 'FLAG_DOCUMENT_13', 'FLAG_DOCUMENT_14', 'FLAG_DOCUMENT_15', 'FLAG_DOCUMENT_16', 'FLAG_DOCUMENT_17', 'FLAG_DOCUMENT_18', 'FLAG_DOCUMENT_19', 'FLAG_DOCUMENT_20', 'FLAG_DOCUMENT_21', 'AMT_REQ_CREDIT_BUREAU_HOUR', 'AMT_REQ_CREDIT_BUREAU_DAY', 'AMT_REQ_CREDIT_BUREAU_WEEK', 'AMT_REQ_CREDIT_BUREAU_MON', 'AMT_REQ_CREDIT_BUREAU_QRT', 'AMT_REQ_CREDIT_BUREAU_YEAR'] \n",
      "\n",
      "\n",
      " df_column_dtypes:  \n",
      " SK_ID_CURR                      int64\n",
      "TARGET                          int64\n",
      "NAME_CONTRACT_TYPE             object\n",
      "CODE_GENDER                    object\n",
      "FLAG_OWN_CAR                   object\n",
      "FLAG_OWN_REALTY                object\n",
      "CNT_CHILDREN                    int64\n",
      "AMT_INCOME_TOTAL              float64\n",
      "AMT_CREDIT                    float64\n",
      "AMT_ANNUITY                   float64\n",
      "AMT_GOODS_PRICE               float64\n",
      "NAME_TYPE_SUITE                object\n",
      "NAME_INCOME_TYPE               object\n",
      "NAME_EDUCATION_TYPE            object\n",
      "NAME_FAMILY_STATUS             object\n",
      "NAME_HOUSING_TYPE              object\n",
      "REGION_POPULATION_RELATIVE    float64\n",
      "DAYS_BIRTH                      int64\n",
      "DAYS_EMPLOYED                   int64\n",
      "DAYS_REGISTRATION             float64\n",
      "DAYS_ID_PUBLISH                 int64\n",
      "OWN_CAR_AGE                   float64\n",
      "FLAG_MOBIL                      int64\n",
      "FLAG_EMP_PHONE                  int64\n",
      "FLAG_WORK_PHONE                 int64\n",
      "FLAG_CONT_MOBILE                int64\n",
      "FLAG_PHONE                      int64\n",
      "FLAG_EMAIL                      int64\n",
      "OCCUPATION_TYPE                object\n",
      "CNT_FAM_MEMBERS               float64\n",
      "                               ...   \n",
      "DEF_30_CNT_SOCIAL_CIRCLE      float64\n",
      "OBS_60_CNT_SOCIAL_CIRCLE      float64\n",
      "DEF_60_CNT_SOCIAL_CIRCLE      float64\n",
      "DAYS_LAST_PHONE_CHANGE        float64\n",
      "FLAG_DOCUMENT_2                 int64\n",
      "FLAG_DOCUMENT_3                 int64\n",
      "FLAG_DOCUMENT_4                 int64\n",
      "FLAG_DOCUMENT_5                 int64\n",
      "FLAG_DOCUMENT_6                 int64\n",
      "FLAG_DOCUMENT_7                 int64\n",
      "FLAG_DOCUMENT_8                 int64\n",
      "FLAG_DOCUMENT_9                 int64\n",
      "FLAG_DOCUMENT_10                int64\n",
      "FLAG_DOCUMENT_11                int64\n",
      "FLAG_DOCUMENT_12                int64\n",
      "FLAG_DOCUMENT_13                int64\n",
      "FLAG_DOCUMENT_14                int64\n",
      "FLAG_DOCUMENT_15                int64\n",
      "FLAG_DOCUMENT_16                int64\n",
      "FLAG_DOCUMENT_17                int64\n",
      "FLAG_DOCUMENT_18                int64\n",
      "FLAG_DOCUMENT_19                int64\n",
      "FLAG_DOCUMENT_20                int64\n",
      "FLAG_DOCUMENT_21                int64\n",
      "AMT_REQ_CREDIT_BUREAU_HOUR    float64\n",
      "AMT_REQ_CREDIT_BUREAU_DAY     float64\n",
      "AMT_REQ_CREDIT_BUREAU_WEEK    float64\n",
      "AMT_REQ_CREDIT_BUREAU_MON     float64\n",
      "AMT_REQ_CREDIT_BUREAU_QRT     float64\n",
      "AMT_REQ_CREDIT_BUREAU_YEAR    float64\n",
      "Length: 122, dtype: object \n",
      "\n",
      "\n",
      " df_column_dtype_groups:  \n",
      " {dtype('int64'): Index(['SK_ID_CURR', 'TARGET', 'CNT_CHILDREN', 'DAYS_BIRTH', 'DAYS_EMPLOYED',\n",
      "       'DAYS_ID_PUBLISH', 'FLAG_MOBIL', 'FLAG_EMP_PHONE', 'FLAG_WORK_PHONE',\n",
      "       'FLAG_CONT_MOBILE', 'FLAG_PHONE', 'FLAG_EMAIL', 'REGION_RATING_CLIENT',\n",
      "       'REGION_RATING_CLIENT_W_CITY', 'HOUR_APPR_PROCESS_START',\n",
      "       'REG_REGION_NOT_LIVE_REGION', 'REG_REGION_NOT_WORK_REGION',\n",
      "       'LIVE_REGION_NOT_WORK_REGION', 'REG_CITY_NOT_LIVE_CITY',\n",
      "       'REG_CITY_NOT_WORK_CITY', 'LIVE_CITY_NOT_WORK_CITY', 'FLAG_DOCUMENT_2',\n",
      "       'FLAG_DOCUMENT_3', 'FLAG_DOCUMENT_4', 'FLAG_DOCUMENT_5',\n",
      "       'FLAG_DOCUMENT_6', 'FLAG_DOCUMENT_7', 'FLAG_DOCUMENT_8',\n",
      "       'FLAG_DOCUMENT_9', 'FLAG_DOCUMENT_10', 'FLAG_DOCUMENT_11',\n",
      "       'FLAG_DOCUMENT_12', 'FLAG_DOCUMENT_13', 'FLAG_DOCUMENT_14',\n",
      "       'FLAG_DOCUMENT_15', 'FLAG_DOCUMENT_16', 'FLAG_DOCUMENT_17',\n",
      "       'FLAG_DOCUMENT_18', 'FLAG_DOCUMENT_19', 'FLAG_DOCUMENT_20',\n",
      "       'FLAG_DOCUMENT_21'],\n",
      "      dtype='object'), dtype('float64'): Index(['AMT_INCOME_TOTAL', 'AMT_CREDIT', 'AMT_ANNUITY', 'AMT_GOODS_PRICE',\n",
      "       'REGION_POPULATION_RELATIVE', 'DAYS_REGISTRATION', 'OWN_CAR_AGE',\n",
      "       'CNT_FAM_MEMBERS', 'EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3',\n",
      "       'APARTMENTS_AVG', 'BASEMENTAREA_AVG', 'YEARS_BEGINEXPLUATATION_AVG',\n",
      "       'YEARS_BUILD_AVG', 'COMMONAREA_AVG', 'ELEVATORS_AVG', 'ENTRANCES_AVG',\n",
      "       'FLOORSMAX_AVG', 'FLOORSMIN_AVG', 'LANDAREA_AVG',\n",
      "       'LIVINGAPARTMENTS_AVG', 'LIVINGAREA_AVG', 'NONLIVINGAPARTMENTS_AVG',\n",
      "       'NONLIVINGAREA_AVG', 'APARTMENTS_MODE', 'BASEMENTAREA_MODE',\n",
      "       'YEARS_BEGINEXPLUATATION_MODE', 'YEARS_BUILD_MODE', 'COMMONAREA_MODE',\n",
      "       'ELEVATORS_MODE', 'ENTRANCES_MODE', 'FLOORSMAX_MODE', 'FLOORSMIN_MODE',\n",
      "       'LANDAREA_MODE', 'LIVINGAPARTMENTS_MODE', 'LIVINGAREA_MODE',\n",
      "       'NONLIVINGAPARTMENTS_MODE', 'NONLIVINGAREA_MODE', 'APARTMENTS_MEDI',\n",
      "       'BASEMENTAREA_MEDI', 'YEARS_BEGINEXPLUATATION_MEDI', 'YEARS_BUILD_MEDI',\n",
      "       'COMMONAREA_MEDI', 'ELEVATORS_MEDI', 'ENTRANCES_MEDI', 'FLOORSMAX_MEDI',\n",
      "       'FLOORSMIN_MEDI', 'LANDAREA_MEDI', 'LIVINGAPARTMENTS_MEDI',\n",
      "       'LIVINGAREA_MEDI', 'NONLIVINGAPARTMENTS_MEDI', 'NONLIVINGAREA_MEDI',\n",
      "       'TOTALAREA_MODE', 'OBS_30_CNT_SOCIAL_CIRCLE',\n",
      "       'DEF_30_CNT_SOCIAL_CIRCLE', 'OBS_60_CNT_SOCIAL_CIRCLE',\n",
      "       'DEF_60_CNT_SOCIAL_CIRCLE', 'DAYS_LAST_PHONE_CHANGE',\n",
      "       'AMT_REQ_CREDIT_BUREAU_HOUR', 'AMT_REQ_CREDIT_BUREAU_DAY',\n",
      "       'AMT_REQ_CREDIT_BUREAU_WEEK', 'AMT_REQ_CREDIT_BUREAU_MON',\n",
      "       'AMT_REQ_CREDIT_BUREAU_QRT', 'AMT_REQ_CREDIT_BUREAU_YEAR'],\n",
      "      dtype='object'), dtype('O'): Index(['NAME_CONTRACT_TYPE', 'CODE_GENDER', 'FLAG_OWN_CAR', 'FLAG_OWN_REALTY',\n",
      "       'NAME_TYPE_SUITE', 'NAME_INCOME_TYPE', 'NAME_EDUCATION_TYPE',\n",
      "       'NAME_FAMILY_STATUS', 'NAME_HOUSING_TYPE', 'OCCUPATION_TYPE',\n",
      "       'WEEKDAY_APPR_PROCESS_START', 'ORGANIZATION_TYPE', 'FONDKAPREMONT_MODE',\n",
      "       'HOUSETYPE_MODE', 'WALLSMATERIAL_MODE', 'EMERGENCYSTATE_MODE'],\n",
      "      dtype='object')} \n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " df_missing_values_flag:  True \n",
      "\n",
      "\n",
      " df_missing_values_count_total:  9152465 \n",
      "\n",
      "\n",
      " df_missing_values_percentage_total:  0.24395941907129431 \n",
      "\n",
      "\n",
      " df_columns_row_count:  \n",
      " {'SK_ID_CURR': 307511, 'TARGET': 307511, 'NAME_CONTRACT_TYPE': 307511, 'CODE_GENDER': 307511, 'FLAG_OWN_CAR': 307511, 'FLAG_OWN_REALTY': 307511, 'CNT_CHILDREN': 307511, 'AMT_INCOME_TOTAL': 307511, 'AMT_CREDIT': 307511, 'AMT_ANNUITY': 307511, 'AMT_GOODS_PRICE': 307511, 'NAME_TYPE_SUITE': 307511, 'NAME_INCOME_TYPE': 307511, 'NAME_EDUCATION_TYPE': 307511, 'NAME_FAMILY_STATUS': 307511, 'NAME_HOUSING_TYPE': 307511, 'REGION_POPULATION_RELATIVE': 307511, 'DAYS_BIRTH': 307511, 'DAYS_EMPLOYED': 307511, 'DAYS_REGISTRATION': 307511, 'DAYS_ID_PUBLISH': 307511, 'OWN_CAR_AGE': 307511, 'FLAG_MOBIL': 307511, 'FLAG_EMP_PHONE': 307511, 'FLAG_WORK_PHONE': 307511, 'FLAG_CONT_MOBILE': 307511, 'FLAG_PHONE': 307511, 'FLAG_EMAIL': 307511, 'OCCUPATION_TYPE': 307511, 'CNT_FAM_MEMBERS': 307511, 'REGION_RATING_CLIENT': 307511, 'REGION_RATING_CLIENT_W_CITY': 307511, 'WEEKDAY_APPR_PROCESS_START': 307511, 'HOUR_APPR_PROCESS_START': 307511, 'REG_REGION_NOT_LIVE_REGION': 307511, 'REG_REGION_NOT_WORK_REGION': 307511, 'LIVE_REGION_NOT_WORK_REGION': 307511, 'REG_CITY_NOT_LIVE_CITY': 307511, 'REG_CITY_NOT_WORK_CITY': 307511, 'LIVE_CITY_NOT_WORK_CITY': 307511, 'ORGANIZATION_TYPE': 307511, 'EXT_SOURCE_1': 307511, 'EXT_SOURCE_2': 307511, 'EXT_SOURCE_3': 307511, 'APARTMENTS_AVG': 307511, 'BASEMENTAREA_AVG': 307511, 'YEARS_BEGINEXPLUATATION_AVG': 307511, 'YEARS_BUILD_AVG': 307511, 'COMMONAREA_AVG': 307511, 'ELEVATORS_AVG': 307511, 'ENTRANCES_AVG': 307511, 'FLOORSMAX_AVG': 307511, 'FLOORSMIN_AVG': 307511, 'LANDAREA_AVG': 307511, 'LIVINGAPARTMENTS_AVG': 307511, 'LIVINGAREA_AVG': 307511, 'NONLIVINGAPARTMENTS_AVG': 307511, 'NONLIVINGAREA_AVG': 307511, 'APARTMENTS_MODE': 307511, 'BASEMENTAREA_MODE': 307511, 'YEARS_BEGINEXPLUATATION_MODE': 307511, 'YEARS_BUILD_MODE': 307511, 'COMMONAREA_MODE': 307511, 'ELEVATORS_MODE': 307511, 'ENTRANCES_MODE': 307511, 'FLOORSMAX_MODE': 307511, 'FLOORSMIN_MODE': 307511, 'LANDAREA_MODE': 307511, 'LIVINGAPARTMENTS_MODE': 307511, 'LIVINGAREA_MODE': 307511, 'NONLIVINGAPARTMENTS_MODE': 307511, 'NONLIVINGAREA_MODE': 307511, 'APARTMENTS_MEDI': 307511, 'BASEMENTAREA_MEDI': 307511, 'YEARS_BEGINEXPLUATATION_MEDI': 307511, 'YEARS_BUILD_MEDI': 307511, 'COMMONAREA_MEDI': 307511, 'ELEVATORS_MEDI': 307511, 'ENTRANCES_MEDI': 307511, 'FLOORSMAX_MEDI': 307511, 'FLOORSMIN_MEDI': 307511, 'LANDAREA_MEDI': 307511, 'LIVINGAPARTMENTS_MEDI': 307511, 'LIVINGAREA_MEDI': 307511, 'NONLIVINGAPARTMENTS_MEDI': 307511, 'NONLIVINGAREA_MEDI': 307511, 'FONDKAPREMONT_MODE': 307511, 'HOUSETYPE_MODE': 307511, 'TOTALAREA_MODE': 307511, 'WALLSMATERIAL_MODE': 307511, 'EMERGENCYSTATE_MODE': 307511, 'OBS_30_CNT_SOCIAL_CIRCLE': 307511, 'DEF_30_CNT_SOCIAL_CIRCLE': 307511, 'OBS_60_CNT_SOCIAL_CIRCLE': 307511, 'DEF_60_CNT_SOCIAL_CIRCLE': 307511, 'DAYS_LAST_PHONE_CHANGE': 307511, 'FLAG_DOCUMENT_2': 307511, 'FLAG_DOCUMENT_3': 307511, 'FLAG_DOCUMENT_4': 307511, 'FLAG_DOCUMENT_5': 307511, 'FLAG_DOCUMENT_6': 307511, 'FLAG_DOCUMENT_7': 307511, 'FLAG_DOCUMENT_8': 307511, 'FLAG_DOCUMENT_9': 307511, 'FLAG_DOCUMENT_10': 307511, 'FLAG_DOCUMENT_11': 307511, 'FLAG_DOCUMENT_12': 307511, 'FLAG_DOCUMENT_13': 307511, 'FLAG_DOCUMENT_14': 307511, 'FLAG_DOCUMENT_15': 307511, 'FLAG_DOCUMENT_16': 307511, 'FLAG_DOCUMENT_17': 307511, 'FLAG_DOCUMENT_18': 307511, 'FLAG_DOCUMENT_19': 307511, 'FLAG_DOCUMENT_20': 307511, 'FLAG_DOCUMENT_21': 307511, 'AMT_REQ_CREDIT_BUREAU_HOUR': 307511, 'AMT_REQ_CREDIT_BUREAU_DAY': 307511, 'AMT_REQ_CREDIT_BUREAU_WEEK': 307511, 'AMT_REQ_CREDIT_BUREAU_MON': 307511, 'AMT_REQ_CREDIT_BUREAU_QRT': 307511, 'AMT_REQ_CREDIT_BUREAU_YEAR': 307511} \n",
      "\n",
      "\n",
      " df_columns_number_of_unique_values:  \n",
      " {'SK_ID_CURR': 307511, 'TARGET': 2, 'NAME_CONTRACT_TYPE': 2, 'CODE_GENDER': 3, 'FLAG_OWN_CAR': 2, 'FLAG_OWN_REALTY': 2, 'CNT_CHILDREN': 15, 'AMT_INCOME_TOTAL': 2548, 'AMT_CREDIT': 5603, 'AMT_ANNUITY': 13672, 'AMT_GOODS_PRICE': 1002, 'NAME_TYPE_SUITE': 7, 'NAME_INCOME_TYPE': 8, 'NAME_EDUCATION_TYPE': 5, 'NAME_FAMILY_STATUS': 6, 'NAME_HOUSING_TYPE': 6, 'REGION_POPULATION_RELATIVE': 81, 'DAYS_BIRTH': 17460, 'DAYS_EMPLOYED': 12574, 'DAYS_REGISTRATION': 15688, 'DAYS_ID_PUBLISH': 6168, 'OWN_CAR_AGE': 62, 'FLAG_MOBIL': 2, 'FLAG_EMP_PHONE': 2, 'FLAG_WORK_PHONE': 2, 'FLAG_CONT_MOBILE': 2, 'FLAG_PHONE': 2, 'FLAG_EMAIL': 2, 'OCCUPATION_TYPE': 18, 'CNT_FAM_MEMBERS': 17, 'REGION_RATING_CLIENT': 3, 'REGION_RATING_CLIENT_W_CITY': 3, 'WEEKDAY_APPR_PROCESS_START': 7, 'HOUR_APPR_PROCESS_START': 24, 'REG_REGION_NOT_LIVE_REGION': 2, 'REG_REGION_NOT_WORK_REGION': 2, 'LIVE_REGION_NOT_WORK_REGION': 2, 'REG_CITY_NOT_LIVE_CITY': 2, 'REG_CITY_NOT_WORK_CITY': 2, 'LIVE_CITY_NOT_WORK_CITY': 2, 'ORGANIZATION_TYPE': 58, 'EXT_SOURCE_1': 114584, 'EXT_SOURCE_2': 119831, 'EXT_SOURCE_3': 814, 'APARTMENTS_AVG': 2339, 'BASEMENTAREA_AVG': 3780, 'YEARS_BEGINEXPLUATATION_AVG': 285, 'YEARS_BUILD_AVG': 149, 'COMMONAREA_AVG': 3181, 'ELEVATORS_AVG': 257, 'ENTRANCES_AVG': 285, 'FLOORSMAX_AVG': 403, 'FLOORSMIN_AVG': 305, 'LANDAREA_AVG': 3527, 'LIVINGAPARTMENTS_AVG': 1868, 'LIVINGAREA_AVG': 5199, 'NONLIVINGAPARTMENTS_AVG': 386, 'NONLIVINGAREA_AVG': 3290, 'APARTMENTS_MODE': 760, 'BASEMENTAREA_MODE': 3841, 'YEARS_BEGINEXPLUATATION_MODE': 221, 'YEARS_BUILD_MODE': 154, 'COMMONAREA_MODE': 3128, 'ELEVATORS_MODE': 26, 'ENTRANCES_MODE': 30, 'FLOORSMAX_MODE': 25, 'FLOORSMIN_MODE': 25, 'LANDAREA_MODE': 3563, 'LIVINGAPARTMENTS_MODE': 736, 'LIVINGAREA_MODE': 5301, 'NONLIVINGAPARTMENTS_MODE': 167, 'NONLIVINGAREA_MODE': 3327, 'APARTMENTS_MEDI': 1148, 'BASEMENTAREA_MEDI': 3772, 'YEARS_BEGINEXPLUATATION_MEDI': 245, 'YEARS_BUILD_MEDI': 151, 'COMMONAREA_MEDI': 3202, 'ELEVATORS_MEDI': 46, 'ENTRANCES_MEDI': 46, 'FLOORSMAX_MEDI': 49, 'FLOORSMIN_MEDI': 47, 'LANDAREA_MEDI': 3560, 'LIVINGAPARTMENTS_MEDI': 1097, 'LIVINGAREA_MEDI': 5281, 'NONLIVINGAPARTMENTS_MEDI': 214, 'NONLIVINGAREA_MEDI': 3323, 'FONDKAPREMONT_MODE': 4, 'HOUSETYPE_MODE': 3, 'TOTALAREA_MODE': 5116, 'WALLSMATERIAL_MODE': 7, 'EMERGENCYSTATE_MODE': 2, 'OBS_30_CNT_SOCIAL_CIRCLE': 33, 'DEF_30_CNT_SOCIAL_CIRCLE': 10, 'OBS_60_CNT_SOCIAL_CIRCLE': 33, 'DEF_60_CNT_SOCIAL_CIRCLE': 9, 'DAYS_LAST_PHONE_CHANGE': 3773, 'FLAG_DOCUMENT_2': 2, 'FLAG_DOCUMENT_3': 2, 'FLAG_DOCUMENT_4': 2, 'FLAG_DOCUMENT_5': 2, 'FLAG_DOCUMENT_6': 2, 'FLAG_DOCUMENT_7': 2, 'FLAG_DOCUMENT_8': 2, 'FLAG_DOCUMENT_9': 2, 'FLAG_DOCUMENT_10': 2, 'FLAG_DOCUMENT_11': 2, 'FLAG_DOCUMENT_12': 2, 'FLAG_DOCUMENT_13': 2, 'FLAG_DOCUMENT_14': 2, 'FLAG_DOCUMENT_15': 2, 'FLAG_DOCUMENT_16': 2, 'FLAG_DOCUMENT_17': 2, 'FLAG_DOCUMENT_18': 2, 'FLAG_DOCUMENT_19': 2, 'FLAG_DOCUMENT_20': 2, 'FLAG_DOCUMENT_21': 2, 'AMT_REQ_CREDIT_BUREAU_HOUR': 5, 'AMT_REQ_CREDIT_BUREAU_DAY': 9, 'AMT_REQ_CREDIT_BUREAU_WEEK': 9, 'AMT_REQ_CREDIT_BUREAU_MON': 24, 'AMT_REQ_CREDIT_BUREAU_QRT': 11, 'AMT_REQ_CREDIT_BUREAU_YEAR': 25} \n",
      "\n",
      "\n",
      " df_columns_percentage_of_unique_values:  \n",
      " {'SK_ID_CURR': 1.0, 'TARGET': 6.503832383231819e-06, 'NAME_CONTRACT_TYPE': 6.503832383231819e-06, 'CODE_GENDER': 9.755748574847729e-06, 'FLAG_OWN_CAR': 6.503832383231819e-06, 'FLAG_OWN_REALTY': 6.503832383231819e-06, 'CNT_CHILDREN': 4.8778742874238645e-05, 'AMT_INCOME_TOTAL': 0.008285882456237337, 'AMT_CREDIT': 0.01822048642162394, 'AMT_ANNUITY': 0.04446019817177272, 'AMT_GOODS_PRICE': 0.0032584200239991414, 'NAME_TYPE_SUITE': 2.276341334131137e-05, 'NAME_INCOME_TYPE': 2.6015329532927276e-05, 'NAME_EDUCATION_TYPE': 1.6259580958079547e-05, 'NAME_FAMILY_STATUS': 1.9511497149695458e-05, 'NAME_HOUSING_TYPE': 1.9511497149695458e-05, 'REGION_POPULATION_RELATIVE': 0.00026340521152088867, 'DAYS_BIRTH': 0.05677845670561378, 'DAYS_EMPLOYED': 0.040889594193378447, 'DAYS_REGISTRATION': 0.05101606121407039, 'DAYS_ID_PUBLISH': 0.02005781906988693, 'OWN_CAR_AGE': 0.0002016188038801864, 'FLAG_MOBIL': 6.503832383231819e-06, 'FLAG_EMP_PHONE': 6.503832383231819e-06, 'FLAG_WORK_PHONE': 6.503832383231819e-06, 'FLAG_CONT_MOBILE': 6.503832383231819e-06, 'FLAG_PHONE': 6.503832383231819e-06, 'FLAG_EMAIL': 6.503832383231819e-06, 'OCCUPATION_TYPE': 5.8534491449086374e-05, 'CNT_FAM_MEMBERS': 5.528257525747047e-05, 'REGION_RATING_CLIENT': 9.755748574847729e-06, 'REGION_RATING_CLIENT_W_CITY': 9.755748574847729e-06, 'WEEKDAY_APPR_PROCESS_START': 2.276341334131137e-05, 'HOUR_APPR_PROCESS_START': 7.804598859878183e-05, 'REG_REGION_NOT_LIVE_REGION': 6.503832383231819e-06, 'REG_REGION_NOT_WORK_REGION': 6.503832383231819e-06, 'LIVE_REGION_NOT_WORK_REGION': 6.503832383231819e-06, 'REG_CITY_NOT_LIVE_CITY': 6.503832383231819e-06, 'REG_CITY_NOT_WORK_CITY': 6.503832383231819e-06, 'LIVE_CITY_NOT_WORK_CITY': 6.503832383231819e-06, 'ORGANIZATION_TYPE': 0.00018861113911372275, 'EXT_SOURCE_1': 0.3726175649001174, 'EXT_SOURCE_2': 0.3896803691575261, 'EXT_SOURCE_3': 0.0026470597799753506, 'APARTMENTS_AVG': 0.007606231972189612, 'BASEMENTAREA_AVG': 0.01229224320430814, 'YEARS_BEGINEXPLUATATION_AVG': 0.0009267961146105343, 'YEARS_BUILD_AVG': 0.00048453551255077057, 'COMMONAREA_AVG': 0.010344345405530208, 'ELEVATORS_AVG': 0.0008357424612452888, 'ENTRANCES_AVG': 0.0009267961146105343, 'FLOORSMAX_AVG': 0.0013105222252212116, 'FLOORSMIN_AVG': 0.0009918344384428524, 'LANDAREA_AVG': 0.011469508407829314, 'LIVINGAPARTMENTS_AVG': 0.006074579445938519, 'LIVINGAREA_AVG': 0.016906712280211116, 'NONLIVINGAPARTMENTS_AVG': 0.001255239649963741, 'NONLIVINGAREA_AVG': 0.010698804270416343, 'APARTMENTS_MODE': 0.0024714563056280913, 'BASEMENTAREA_MODE': 0.01249061009199671, 'YEARS_BEGINEXPLUATATION_MODE': 0.000718673478347116, 'YEARS_BUILD_MODE': 0.0005007950935088501, 'COMMONAREA_MODE': 0.010171993847374565, 'ELEVATORS_MODE': 8.454982098201365e-05, 'ENTRANCES_MODE': 9.755748574847729e-05, 'FLOORSMAX_MODE': 8.129790479039775e-05, 'FLOORSMIN_MODE': 8.129790479039775e-05, 'LANDAREA_MODE': 0.011586577390727486, 'LIVINGAPARTMENTS_MODE': 0.0023934103170293094, 'LIVINGAREA_MODE': 0.017238407731755938, 'NONLIVINGAPARTMENTS_MODE': 0.000543070003999857, 'NONLIVINGAREA_MODE': 0.010819125169506132, 'APARTMENTS_MEDI': 0.0037331997879750645, 'BASEMENTAREA_MEDI': 0.012266227874775212, 'YEARS_BEGINEXPLUATATION_MEDI': 0.0007967194669458978, 'YEARS_BUILD_MEDI': 0.0004910393449340023, 'COMMONAREA_MEDI': 0.010412635645554143, 'ELEVATORS_MEDI': 0.00014958814481433184, 'ENTRANCES_MEDI': 0.00014958814481433184, 'FLOORSMAX_MEDI': 0.00015934389338917958, 'FLOORSMIN_MEDI': 0.00015284006100594775, 'LANDAREA_MEDI': 0.011576821642152638, 'LIVINGAPARTMENTS_MEDI': 0.003567352062202653, 'LIVINGAREA_MEDI': 0.01717336940792362, 'NONLIVINGAPARTMENTS_MEDI': 0.0006959100650058047, 'NONLIVINGAREA_MEDI': 0.010806117504739667, 'FONDKAPREMONT_MODE': 1.3007664766463638e-05, 'HOUSETYPE_MODE': 9.755748574847729e-06, 'TOTALAREA_MODE': 0.016636803236306993, 'WALLSMATERIAL_MODE': 2.276341334131137e-05, 'EMERGENCYSTATE_MODE': 6.503832383231819e-06, 'OBS_30_CNT_SOCIAL_CIRCLE': 0.00010731323432332502, 'DEF_30_CNT_SOCIAL_CIRCLE': 3.2519161916159095e-05, 'OBS_60_CNT_SOCIAL_CIRCLE': 0.00010731323432332502, 'DEF_60_CNT_SOCIAL_CIRCLE': 2.9267245724543187e-05, 'DAYS_LAST_PHONE_CHANGE': 0.012269479790966827, 'FLAG_DOCUMENT_2': 6.503832383231819e-06, 'FLAG_DOCUMENT_3': 6.503832383231819e-06, 'FLAG_DOCUMENT_4': 6.503832383231819e-06, 'FLAG_DOCUMENT_5': 6.503832383231819e-06, 'FLAG_DOCUMENT_6': 6.503832383231819e-06, 'FLAG_DOCUMENT_7': 6.503832383231819e-06, 'FLAG_DOCUMENT_8': 6.503832383231819e-06, 'FLAG_DOCUMENT_9': 6.503832383231819e-06, 'FLAG_DOCUMENT_10': 6.503832383231819e-06, 'FLAG_DOCUMENT_11': 6.503832383231819e-06, 'FLAG_DOCUMENT_12': 6.503832383231819e-06, 'FLAG_DOCUMENT_13': 6.503832383231819e-06, 'FLAG_DOCUMENT_14': 6.503832383231819e-06, 'FLAG_DOCUMENT_15': 6.503832383231819e-06, 'FLAG_DOCUMENT_16': 6.503832383231819e-06, 'FLAG_DOCUMENT_17': 6.503832383231819e-06, 'FLAG_DOCUMENT_18': 6.503832383231819e-06, 'FLAG_DOCUMENT_19': 6.503832383231819e-06, 'FLAG_DOCUMENT_20': 6.503832383231819e-06, 'FLAG_DOCUMENT_21': 6.503832383231819e-06, 'AMT_REQ_CREDIT_BUREAU_HOUR': 1.6259580958079547e-05, 'AMT_REQ_CREDIT_BUREAU_DAY': 2.9267245724543187e-05, 'AMT_REQ_CREDIT_BUREAU_WEEK': 2.9267245724543187e-05, 'AMT_REQ_CREDIT_BUREAU_MON': 7.804598859878183e-05, 'AMT_REQ_CREDIT_BUREAU_QRT': 3.577107810777501e-05, 'AMT_REQ_CREDIT_BUREAU_YEAR': 8.129790479039775e-05} \n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " df_columns_missing_values_flag:  \n",
      " {'SK_ID_CURR': False, 'TARGET': False, 'NAME_CONTRACT_TYPE': False, 'CODE_GENDER': False, 'FLAG_OWN_CAR': False, 'FLAG_OWN_REALTY': False, 'CNT_CHILDREN': False, 'AMT_INCOME_TOTAL': False, 'AMT_CREDIT': False, 'AMT_ANNUITY': True, 'AMT_GOODS_PRICE': True, 'NAME_TYPE_SUITE': True, 'NAME_INCOME_TYPE': False, 'NAME_EDUCATION_TYPE': False, 'NAME_FAMILY_STATUS': False, 'NAME_HOUSING_TYPE': False, 'REGION_POPULATION_RELATIVE': False, 'DAYS_BIRTH': False, 'DAYS_EMPLOYED': False, 'DAYS_REGISTRATION': False, 'DAYS_ID_PUBLISH': False, 'OWN_CAR_AGE': True, 'FLAG_MOBIL': False, 'FLAG_EMP_PHONE': False, 'FLAG_WORK_PHONE': False, 'FLAG_CONT_MOBILE': False, 'FLAG_PHONE': False, 'FLAG_EMAIL': False, 'OCCUPATION_TYPE': True, 'CNT_FAM_MEMBERS': True, 'REGION_RATING_CLIENT': False, 'REGION_RATING_CLIENT_W_CITY': False, 'WEEKDAY_APPR_PROCESS_START': False, 'HOUR_APPR_PROCESS_START': False, 'REG_REGION_NOT_LIVE_REGION': False, 'REG_REGION_NOT_WORK_REGION': False, 'LIVE_REGION_NOT_WORK_REGION': False, 'REG_CITY_NOT_LIVE_CITY': False, 'REG_CITY_NOT_WORK_CITY': False, 'LIVE_CITY_NOT_WORK_CITY': False, 'ORGANIZATION_TYPE': False, 'EXT_SOURCE_1': True, 'EXT_SOURCE_2': True, 'EXT_SOURCE_3': True, 'APARTMENTS_AVG': True, 'BASEMENTAREA_AVG': True, 'YEARS_BEGINEXPLUATATION_AVG': True, 'YEARS_BUILD_AVG': True, 'COMMONAREA_AVG': True, 'ELEVATORS_AVG': True, 'ENTRANCES_AVG': True, 'FLOORSMAX_AVG': True, 'FLOORSMIN_AVG': True, 'LANDAREA_AVG': True, 'LIVINGAPARTMENTS_AVG': True, 'LIVINGAREA_AVG': True, 'NONLIVINGAPARTMENTS_AVG': True, 'NONLIVINGAREA_AVG': True, 'APARTMENTS_MODE': True, 'BASEMENTAREA_MODE': True, 'YEARS_BEGINEXPLUATATION_MODE': True, 'YEARS_BUILD_MODE': True, 'COMMONAREA_MODE': True, 'ELEVATORS_MODE': True, 'ENTRANCES_MODE': True, 'FLOORSMAX_MODE': True, 'FLOORSMIN_MODE': True, 'LANDAREA_MODE': True, 'LIVINGAPARTMENTS_MODE': True, 'LIVINGAREA_MODE': True, 'NONLIVINGAPARTMENTS_MODE': True, 'NONLIVINGAREA_MODE': True, 'APARTMENTS_MEDI': True, 'BASEMENTAREA_MEDI': True, 'YEARS_BEGINEXPLUATATION_MEDI': True, 'YEARS_BUILD_MEDI': True, 'COMMONAREA_MEDI': True, 'ELEVATORS_MEDI': True, 'ENTRANCES_MEDI': True, 'FLOORSMAX_MEDI': True, 'FLOORSMIN_MEDI': True, 'LANDAREA_MEDI': True, 'LIVINGAPARTMENTS_MEDI': True, 'LIVINGAREA_MEDI': True, 'NONLIVINGAPARTMENTS_MEDI': True, 'NONLIVINGAREA_MEDI': True, 'FONDKAPREMONT_MODE': True, 'HOUSETYPE_MODE': True, 'TOTALAREA_MODE': True, 'WALLSMATERIAL_MODE': True, 'EMERGENCYSTATE_MODE': True, 'OBS_30_CNT_SOCIAL_CIRCLE': True, 'DEF_30_CNT_SOCIAL_CIRCLE': True, 'OBS_60_CNT_SOCIAL_CIRCLE': True, 'DEF_60_CNT_SOCIAL_CIRCLE': True, 'DAYS_LAST_PHONE_CHANGE': True, 'FLAG_DOCUMENT_2': False, 'FLAG_DOCUMENT_3': False, 'FLAG_DOCUMENT_4': False, 'FLAG_DOCUMENT_5': False, 'FLAG_DOCUMENT_6': False, 'FLAG_DOCUMENT_7': False, 'FLAG_DOCUMENT_8': False, 'FLAG_DOCUMENT_9': False, 'FLAG_DOCUMENT_10': False, 'FLAG_DOCUMENT_11': False, 'FLAG_DOCUMENT_12': False, 'FLAG_DOCUMENT_13': False, 'FLAG_DOCUMENT_14': False, 'FLAG_DOCUMENT_15': False, 'FLAG_DOCUMENT_16': False, 'FLAG_DOCUMENT_17': False, 'FLAG_DOCUMENT_18': False, 'FLAG_DOCUMENT_19': False, 'FLAG_DOCUMENT_20': False, 'FLAG_DOCUMENT_21': False, 'AMT_REQ_CREDIT_BUREAU_HOUR': True, 'AMT_REQ_CREDIT_BUREAU_DAY': True, 'AMT_REQ_CREDIT_BUREAU_WEEK': True, 'AMT_REQ_CREDIT_BUREAU_MON': True, 'AMT_REQ_CREDIT_BUREAU_QRT': True, 'AMT_REQ_CREDIT_BUREAU_YEAR': True} \n",
      "\n",
      "\n",
      " df_columns_missing_values_count:  \n",
      " {'SK_ID_CURR': 0, 'TARGET': 0, 'NAME_CONTRACT_TYPE': 0, 'CODE_GENDER': 0, 'FLAG_OWN_CAR': 0, 'FLAG_OWN_REALTY': 0, 'CNT_CHILDREN': 0, 'AMT_INCOME_TOTAL': 0, 'AMT_CREDIT': 0, 'AMT_ANNUITY': 12, 'AMT_GOODS_PRICE': 278, 'NAME_TYPE_SUITE': 1292, 'NAME_INCOME_TYPE': 0, 'NAME_EDUCATION_TYPE': 0, 'NAME_FAMILY_STATUS': 0, 'NAME_HOUSING_TYPE': 0, 'REGION_POPULATION_RELATIVE': 0, 'DAYS_BIRTH': 0, 'DAYS_EMPLOYED': 0, 'DAYS_REGISTRATION': 0, 'DAYS_ID_PUBLISH': 0, 'OWN_CAR_AGE': 202929, 'FLAG_MOBIL': 0, 'FLAG_EMP_PHONE': 0, 'FLAG_WORK_PHONE': 0, 'FLAG_CONT_MOBILE': 0, 'FLAG_PHONE': 0, 'FLAG_EMAIL': 0, 'OCCUPATION_TYPE': 96391, 'CNT_FAM_MEMBERS': 2, 'REGION_RATING_CLIENT': 0, 'REGION_RATING_CLIENT_W_CITY': 0, 'WEEKDAY_APPR_PROCESS_START': 0, 'HOUR_APPR_PROCESS_START': 0, 'REG_REGION_NOT_LIVE_REGION': 0, 'REG_REGION_NOT_WORK_REGION': 0, 'LIVE_REGION_NOT_WORK_REGION': 0, 'REG_CITY_NOT_LIVE_CITY': 0, 'REG_CITY_NOT_WORK_CITY': 0, 'LIVE_CITY_NOT_WORK_CITY': 0, 'ORGANIZATION_TYPE': 0, 'EXT_SOURCE_1': 173378, 'EXT_SOURCE_2': 660, 'EXT_SOURCE_3': 60965, 'APARTMENTS_AVG': 156061, 'BASEMENTAREA_AVG': 179943, 'YEARS_BEGINEXPLUATATION_AVG': 150007, 'YEARS_BUILD_AVG': 204488, 'COMMONAREA_AVG': 214865, 'ELEVATORS_AVG': 163891, 'ENTRANCES_AVG': 154828, 'FLOORSMAX_AVG': 153020, 'FLOORSMIN_AVG': 208642, 'LANDAREA_AVG': 182590, 'LIVINGAPARTMENTS_AVG': 210199, 'LIVINGAREA_AVG': 154350, 'NONLIVINGAPARTMENTS_AVG': 213514, 'NONLIVINGAREA_AVG': 169682, 'APARTMENTS_MODE': 156061, 'BASEMENTAREA_MODE': 179943, 'YEARS_BEGINEXPLUATATION_MODE': 150007, 'YEARS_BUILD_MODE': 204488, 'COMMONAREA_MODE': 214865, 'ELEVATORS_MODE': 163891, 'ENTRANCES_MODE': 154828, 'FLOORSMAX_MODE': 153020, 'FLOORSMIN_MODE': 208642, 'LANDAREA_MODE': 182590, 'LIVINGAPARTMENTS_MODE': 210199, 'LIVINGAREA_MODE': 154350, 'NONLIVINGAPARTMENTS_MODE': 213514, 'NONLIVINGAREA_MODE': 169682, 'APARTMENTS_MEDI': 156061, 'BASEMENTAREA_MEDI': 179943, 'YEARS_BEGINEXPLUATATION_MEDI': 150007, 'YEARS_BUILD_MEDI': 204488, 'COMMONAREA_MEDI': 214865, 'ELEVATORS_MEDI': 163891, 'ENTRANCES_MEDI': 154828, 'FLOORSMAX_MEDI': 153020, 'FLOORSMIN_MEDI': 208642, 'LANDAREA_MEDI': 182590, 'LIVINGAPARTMENTS_MEDI': 210199, 'LIVINGAREA_MEDI': 154350, 'NONLIVINGAPARTMENTS_MEDI': 213514, 'NONLIVINGAREA_MEDI': 169682, 'FONDKAPREMONT_MODE': 210295, 'HOUSETYPE_MODE': 154297, 'TOTALAREA_MODE': 148431, 'WALLSMATERIAL_MODE': 156341, 'EMERGENCYSTATE_MODE': 145755, 'OBS_30_CNT_SOCIAL_CIRCLE': 1021, 'DEF_30_CNT_SOCIAL_CIRCLE': 1021, 'OBS_60_CNT_SOCIAL_CIRCLE': 1021, 'DEF_60_CNT_SOCIAL_CIRCLE': 1021, 'DAYS_LAST_PHONE_CHANGE': 1, 'FLAG_DOCUMENT_2': 0, 'FLAG_DOCUMENT_3': 0, 'FLAG_DOCUMENT_4': 0, 'FLAG_DOCUMENT_5': 0, 'FLAG_DOCUMENT_6': 0, 'FLAG_DOCUMENT_7': 0, 'FLAG_DOCUMENT_8': 0, 'FLAG_DOCUMENT_9': 0, 'FLAG_DOCUMENT_10': 0, 'FLAG_DOCUMENT_11': 0, 'FLAG_DOCUMENT_12': 0, 'FLAG_DOCUMENT_13': 0, 'FLAG_DOCUMENT_14': 0, 'FLAG_DOCUMENT_15': 0, 'FLAG_DOCUMENT_16': 0, 'FLAG_DOCUMENT_17': 0, 'FLAG_DOCUMENT_18': 0, 'FLAG_DOCUMENT_19': 0, 'FLAG_DOCUMENT_20': 0, 'FLAG_DOCUMENT_21': 0, 'AMT_REQ_CREDIT_BUREAU_HOUR': 41519, 'AMT_REQ_CREDIT_BUREAU_DAY': 41519, 'AMT_REQ_CREDIT_BUREAU_WEEK': 41519, 'AMT_REQ_CREDIT_BUREAU_MON': 41519, 'AMT_REQ_CREDIT_BUREAU_QRT': 41519, 'AMT_REQ_CREDIT_BUREAU_YEAR': 41519} \n",
      "\n",
      "\n",
      " df_columns_missing_values_percentage:  \n",
      " {'SK_ID_CURR': 0.0, 'TARGET': 0.0, 'NAME_CONTRACT_TYPE': 0.0, 'CODE_GENDER': 0.0, 'FLAG_OWN_CAR': 0.0, 'FLAG_OWN_REALTY': 0.0, 'CNT_CHILDREN': 0.0, 'AMT_INCOME_TOTAL': 0.0, 'AMT_CREDIT': 0.0, 'AMT_ANNUITY': 3.9022994299390916e-05, 'AMT_GOODS_PRICE': 0.0009040327012692228, 'NAME_TYPE_SUITE': 0.004201475719567756, 'NAME_INCOME_TYPE': 0.0, 'NAME_EDUCATION_TYPE': 0.0, 'NAME_FAMILY_STATUS': 0.0, 'NAME_HOUSING_TYPE': 0.0, 'REGION_POPULATION_RELATIVE': 0.0, 'DAYS_BIRTH': 0.0, 'DAYS_EMPLOYED': 0.0, 'DAYS_REGISTRATION': 0.0, 'DAYS_ID_PUBLISH': 0.0, 'OWN_CAR_AGE': 0.6599081008484249, 'FLAG_MOBIL': 0.0, 'FLAG_EMP_PHONE': 0.0, 'FLAG_WORK_PHONE': 0.0, 'FLAG_CONT_MOBILE': 0.0, 'FLAG_PHONE': 0.0, 'FLAG_EMAIL': 0.0, 'OCCUPATION_TYPE': 0.31345545362604915, 'CNT_FAM_MEMBERS': 6.503832383231819e-06, 'REGION_RATING_CLIENT': 0.0, 'REGION_RATING_CLIENT_W_CITY': 0.0, 'WEEKDAY_APPR_PROCESS_START': 0.0, 'HOUR_APPR_PROCESS_START': 0.0, 'REG_REGION_NOT_LIVE_REGION': 0.0, 'REG_REGION_NOT_WORK_REGION': 0.0, 'LIVE_REGION_NOT_WORK_REGION': 0.0, 'REG_CITY_NOT_LIVE_CITY': 0.0, 'REG_CITY_NOT_WORK_CITY': 0.0, 'LIVE_CITY_NOT_WORK_CITY': 0.0, 'ORGANIZATION_TYPE': 0.0, 'EXT_SOURCE_1': 0.5638107254699832, 'EXT_SOURCE_2': 0.0021462646864665006, 'EXT_SOURCE_3': 0.19825307062186392, 'APARTMENTS_AVG': 0.5074972927797705, 'BASEMENTAREA_AVG': 0.5851595552679416, 'YEARS_BEGINEXPLUATATION_AVG': 0.48781019215572774, 'YEARS_BUILD_AVG': 0.6649778381911542, 'COMMONAREA_AVG': 0.6987229725115525, 'ELEVATORS_AVG': 0.532959796560123, 'ENTRANCES_AVG': 0.503487680115508, 'FLOORSMAX_AVG': 0.4976082156410665, 'FLOORSMIN_AVG': 0.6784862980511266, 'LANDAREA_AVG': 0.5937673774271489, 'LIVINGAPARTMENTS_AVG': 0.6835495315614726, 'LIVINGAREA_AVG': 0.5019332641759157, 'NONLIVINGAPARTMENTS_AVG': 0.6943296337366793, 'NONLIVINGAREA_AVG': 0.5517916432257708, 'APARTMENTS_MODE': 0.5074972927797705, 'BASEMENTAREA_MODE': 0.5851595552679416, 'YEARS_BEGINEXPLUATATION_MODE': 0.48781019215572774, 'YEARS_BUILD_MODE': 0.6649778381911542, 'COMMONAREA_MODE': 0.6987229725115525, 'ELEVATORS_MODE': 0.532959796560123, 'ENTRANCES_MODE': 0.503487680115508, 'FLOORSMAX_MODE': 0.4976082156410665, 'FLOORSMIN_MODE': 0.6784862980511266, 'LANDAREA_MODE': 0.5937673774271489, 'LIVINGAPARTMENTS_MODE': 0.6835495315614726, 'LIVINGAREA_MODE': 0.5019332641759157, 'NONLIVINGAPARTMENTS_MODE': 0.6943296337366793, 'NONLIVINGAREA_MODE': 0.5517916432257708, 'APARTMENTS_MEDI': 0.5074972927797705, 'BASEMENTAREA_MEDI': 0.5851595552679416, 'YEARS_BEGINEXPLUATATION_MEDI': 0.48781019215572774, 'YEARS_BUILD_MEDI': 0.6649778381911542, 'COMMONAREA_MEDI': 0.6987229725115525, 'ELEVATORS_MEDI': 0.532959796560123, 'ENTRANCES_MEDI': 0.503487680115508, 'FLOORSMAX_MEDI': 0.4976082156410665, 'FLOORSMIN_MEDI': 0.6784862980511266, 'LANDAREA_MEDI': 0.5937673774271489, 'LIVINGAPARTMENTS_MEDI': 0.6835495315614726, 'LIVINGAREA_MEDI': 0.5019332641759157, 'NONLIVINGAPARTMENTS_MEDI': 0.6943296337366793, 'NONLIVINGAREA_MEDI': 0.5517916432257708, 'FONDKAPREMONT_MODE': 0.6838617155158677, 'HOUSETYPE_MODE': 0.50176091261776, 'TOTALAREA_MODE': 0.4826851722377411, 'WALLSMATERIAL_MODE': 0.508407829313423, 'EMERGENCYSTATE_MODE': 0.47398304450897694, 'OBS_30_CNT_SOCIAL_CIRCLE': 0.0033202064316398437, 'DEF_30_CNT_SOCIAL_CIRCLE': 0.0033202064316398437, 'OBS_60_CNT_SOCIAL_CIRCLE': 0.0033202064316398437, 'DEF_60_CNT_SOCIAL_CIRCLE': 0.0033202064316398437, 'DAYS_LAST_PHONE_CHANGE': 3.2519161916159095e-06, 'FLAG_DOCUMENT_2': 0.0, 'FLAG_DOCUMENT_3': 0.0, 'FLAG_DOCUMENT_4': 0.0, 'FLAG_DOCUMENT_5': 0.0, 'FLAG_DOCUMENT_6': 0.0, 'FLAG_DOCUMENT_7': 0.0, 'FLAG_DOCUMENT_8': 0.0, 'FLAG_DOCUMENT_9': 0.0, 'FLAG_DOCUMENT_10': 0.0, 'FLAG_DOCUMENT_11': 0.0, 'FLAG_DOCUMENT_12': 0.0, 'FLAG_DOCUMENT_13': 0.0, 'FLAG_DOCUMENT_14': 0.0, 'FLAG_DOCUMENT_15': 0.0, 'FLAG_DOCUMENT_16': 0.0, 'FLAG_DOCUMENT_17': 0.0, 'FLAG_DOCUMENT_18': 0.0, 'FLAG_DOCUMENT_19': 0.0, 'FLAG_DOCUMENT_20': 0.0, 'FLAG_DOCUMENT_21': 0.0, 'AMT_REQ_CREDIT_BUREAU_HOUR': 0.13501630835970097, 'AMT_REQ_CREDIT_BUREAU_DAY': 0.13501630835970097, 'AMT_REQ_CREDIT_BUREAU_WEEK': 0.13501630835970097, 'AMT_REQ_CREDIT_BUREAU_MON': 0.13501630835970097, 'AMT_REQ_CREDIT_BUREAU_QRT': 0.13501630835970097, 'AMT_REQ_CREDIT_BUREAU_YEAR': 0.13501630835970097} \n",
      "\n",
      "\n",
      " df_target_column_name:  TARGET \n",
      "\n",
      "\n",
      " df_record_id_column_name:  SK_ID_CURR \n",
      "\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " df_record_key_column_name:  SK_ID_CURR \n",
      "\n",
      "SK_ID_CURR dtype:  int64\n",
      "SK_ID_CURR dtype:  int64\n",
      "TARGET dtype:  int64\n",
      "NAME_CONTRACT_TYPE dtype:  object\n",
      "FLAG_OWN_CAR dtype:  object\n",
      "FLAG_OWN_REALTY dtype:  object\n",
      "FLAG_MOBIL dtype:  int64\n",
      "FLAG_EMP_PHONE dtype:  int64\n",
      "FLAG_WORK_PHONE dtype:  int64\n",
      "FLAG_CONT_MOBILE dtype:  int64\n",
      "FLAG_PHONE dtype:  int64\n",
      "FLAG_EMAIL dtype:  int64\n",
      "REG_REGION_NOT_LIVE_REGION dtype:  int64\n",
      "REG_REGION_NOT_WORK_REGION dtype:  int64\n",
      "LIVE_REGION_NOT_WORK_REGION dtype:  int64\n",
      "REG_CITY_NOT_LIVE_CITY dtype:  int64\n",
      "REG_CITY_NOT_WORK_CITY dtype:  int64\n",
      "LIVE_CITY_NOT_WORK_CITY dtype:  int64\n",
      "EMERGENCYSTATE_MODE dtype:  object\n",
      "FLAG_DOCUMENT_2 dtype:  int64\n",
      "FLAG_DOCUMENT_3 dtype:  int64\n",
      "FLAG_DOCUMENT_4 dtype:  int64\n",
      "FLAG_DOCUMENT_5 dtype:  int64\n",
      "FLAG_DOCUMENT_6 dtype:  int64\n",
      "FLAG_DOCUMENT_7 dtype:  int64\n",
      "FLAG_DOCUMENT_8 dtype:  int64\n",
      "FLAG_DOCUMENT_9 dtype:  int64\n",
      "FLAG_DOCUMENT_10 dtype:  int64\n",
      "FLAG_DOCUMENT_11 dtype:  int64\n",
      "FLAG_DOCUMENT_12 dtype:  int64\n",
      "FLAG_DOCUMENT_13 dtype:  int64\n",
      "FLAG_DOCUMENT_14 dtype:  int64\n",
      "FLAG_DOCUMENT_15 dtype:  int64\n",
      "FLAG_DOCUMENT_16 dtype:  int64\n",
      "FLAG_DOCUMENT_17 dtype:  int64\n",
      "FLAG_DOCUMENT_18 dtype:  int64\n",
      "FLAG_DOCUMENT_19 dtype:  int64\n",
      "FLAG_DOCUMENT_20 dtype:  int64\n",
      "FLAG_DOCUMENT_21 dtype:  int64\n",
      "0    282686\n",
      "1     24825\n",
      "Name: TARGET, dtype: int64\n",
      "Cash loans         278232\n",
      "Revolving loans     29279\n",
      "Name: NAME_CONTRACT_TYPE, dtype: int64\n",
      "N    202924\n",
      "Y    104587\n",
      "Name: FLAG_OWN_CAR, dtype: int64\n",
      "Y    213312\n",
      "N     94199\n",
      "Name: FLAG_OWN_REALTY, dtype: int64\n",
      "1    307510\n",
      "0         1\n",
      "Name: FLAG_MOBIL, dtype: int64\n",
      "1    252125\n",
      "0     55386\n",
      "Name: FLAG_EMP_PHONE, dtype: int64\n",
      "0    246203\n",
      "1     61308\n",
      "Name: FLAG_WORK_PHONE, dtype: int64\n",
      "1    306937\n",
      "0       574\n",
      "Name: FLAG_CONT_MOBILE, dtype: int64\n",
      "0    221080\n",
      "1     86431\n",
      "Name: FLAG_PHONE, dtype: int64\n",
      "0    290069\n",
      "1     17442\n",
      "Name: FLAG_EMAIL, dtype: int64\n",
      "0    302854\n",
      "1      4657\n",
      "Name: REG_REGION_NOT_LIVE_REGION, dtype: int64\n",
      "0    291899\n",
      "1     15612\n",
      "Name: REG_REGION_NOT_WORK_REGION, dtype: int64\n",
      "0    295008\n",
      "1     12503\n",
      "Name: LIVE_REGION_NOT_WORK_REGION, dtype: int64\n",
      "0    283472\n",
      "1     24039\n",
      "Name: REG_CITY_NOT_LIVE_CITY, dtype: int64\n",
      "0    236644\n",
      "1     70867\n",
      "Name: REG_CITY_NOT_WORK_CITY, dtype: int64\n",
      "0    252296\n",
      "1     55215\n",
      "Name: LIVE_CITY_NOT_WORK_CITY, dtype: int64\n",
      "No     159428\n",
      "NaN    145755\n",
      "Yes      2328\n",
      "Name: EMERGENCYSTATE_MODE, dtype: int64\n",
      "0    307498\n",
      "1        13\n",
      "Name: FLAG_DOCUMENT_2, dtype: int64\n",
      "1    218340\n",
      "0     89171\n",
      "Name: FLAG_DOCUMENT_3, dtype: int64\n",
      "0    307486\n",
      "1        25\n",
      "Name: FLAG_DOCUMENT_4, dtype: int64\n",
      "0    302863\n",
      "1      4648\n",
      "Name: FLAG_DOCUMENT_5, dtype: int64\n",
      "0    280433\n",
      "1     27078\n",
      "Name: FLAG_DOCUMENT_6, dtype: int64\n",
      "0    307452\n",
      "1        59\n",
      "Name: FLAG_DOCUMENT_7, dtype: int64\n",
      "0    282487\n",
      "1     25024\n",
      "Name: FLAG_DOCUMENT_8, dtype: int64\n",
      "0    306313\n",
      "1      1198\n",
      "Name: FLAG_DOCUMENT_9, dtype: int64\n",
      "0    307504\n",
      "1         7\n",
      "Name: FLAG_DOCUMENT_10, dtype: int64\n",
      "0    306308\n",
      "1      1203\n",
      "Name: FLAG_DOCUMENT_11, dtype: int64\n",
      "0    307509\n",
      "1         2\n",
      "Name: FLAG_DOCUMENT_12, dtype: int64\n",
      "0    306427\n",
      "1      1084\n",
      "Name: FLAG_DOCUMENT_13, dtype: int64\n",
      "0    306608\n",
      "1       903\n",
      "Name: FLAG_DOCUMENT_14, dtype: int64\n",
      "0    307139\n",
      "1       372\n",
      "Name: FLAG_DOCUMENT_15, dtype: int64\n",
      "0    304458\n",
      "1      3053\n",
      "Name: FLAG_DOCUMENT_16, dtype: int64\n",
      "0    307429\n",
      "1        82\n",
      "Name: FLAG_DOCUMENT_17, dtype: int64\n",
      "0    305011\n",
      "1      2500\n",
      "Name: FLAG_DOCUMENT_18, dtype: int64\n",
      "0    307328\n",
      "1       183\n",
      "Name: FLAG_DOCUMENT_19, dtype: int64\n",
      "0    307355\n",
      "1       156\n",
      "Name: FLAG_DOCUMENT_20, dtype: int64\n",
      "0    307408\n",
      "1       103\n",
      "Name: FLAG_DOCUMENT_21, dtype: int64\n",
      "CODE_GENDER dtype:  object\n",
      "REGION_RATING_CLIENT dtype:  int64\n",
      "REGION_RATING_CLIENT_W_CITY dtype:  int64\n",
      "HOUSETYPE_MODE dtype:  object\n",
      "F      202448\n",
      "M      105059\n",
      "XNA         4\n",
      "Name: CODE_GENDER, dtype: int64\n",
      "2    226984\n",
      "3     48330\n",
      "1     32197\n",
      "Name: REGION_RATING_CLIENT, dtype: int64\n",
      "2    229484\n",
      "3     43860\n",
      "1     34167\n",
      "Name: REGION_RATING_CLIENT_W_CITY, dtype: int64\n",
      "NaN                 154297\n",
      "block of flats      150503\n",
      "specific housing      1499\n",
      "terraced house        1212\n",
      "Name: HOUSETYPE_MODE, dtype: int64\n",
      "OWN_CAR_AGE 0.6599081008484249\n",
      "YEARS_BUILD_AVG 0.6649778381911542\n",
      "COMMONAREA_AVG 0.6987229725115525\n",
      "FLOORSMIN_AVG 0.6784862980511266\n",
      "LIVINGAPARTMENTS_AVG 0.6835495315614726\n",
      "NONLIVINGAPARTMENTS_AVG 0.6943296337366793\n",
      "YEARS_BUILD_MODE 0.6649778381911542\n",
      "COMMONAREA_MODE 0.6987229725115525\n",
      "FLOORSMIN_MODE 0.6784862980511266\n",
      "LIVINGAPARTMENTS_MODE 0.6835495315614726\n",
      "NONLIVINGAPARTMENTS_MODE 0.6943296337366793\n",
      "YEARS_BUILD_MEDI 0.6649778381911542\n",
      "COMMONAREA_MEDI 0.6987229725115525\n",
      "FLOORSMIN_MEDI 0.6784862980511266\n",
      "LIVINGAPARTMENTS_MEDI 0.6835495315614726\n",
      "NONLIVINGAPARTMENTS_MEDI 0.6943296337366793\n",
      "FONDKAPREMONT_MODE 0.6838617155158677\n",
      "OWN_CAR_AGE 0.6599081008484249\n",
      "OCCUPATION_TYPE 0.31345545362604915\n",
      "EXT_SOURCE_1 0.5638107254699832\n",
      "APARTMENTS_AVG 0.5074972927797705\n",
      "BASEMENTAREA_AVG 0.5851595552679416\n",
      "YEARS_BEGINEXPLUATATION_AVG 0.48781019215572774\n",
      "YEARS_BUILD_AVG 0.6649778381911542\n",
      "COMMONAREA_AVG 0.6987229725115525\n",
      "ELEVATORS_AVG 0.532959796560123\n",
      "ENTRANCES_AVG 0.503487680115508\n",
      "FLOORSMAX_AVG 0.4976082156410665\n",
      "FLOORSMIN_AVG 0.6784862980511266\n",
      "LANDAREA_AVG 0.5937673774271489\n",
      "LIVINGAPARTMENTS_AVG 0.6835495315614726\n",
      "LIVINGAREA_AVG 0.5019332641759157\n",
      "NONLIVINGAPARTMENTS_AVG 0.6943296337366793\n",
      "NONLIVINGAREA_AVG 0.5517916432257708\n",
      "APARTMENTS_MODE 0.5074972927797705\n",
      "BASEMENTAREA_MODE 0.5851595552679416\n",
      "YEARS_BEGINEXPLUATATION_MODE 0.48781019215572774\n",
      "YEARS_BUILD_MODE 0.6649778381911542\n",
      "COMMONAREA_MODE 0.6987229725115525\n",
      "ELEVATORS_MODE 0.532959796560123\n",
      "ENTRANCES_MODE 0.503487680115508\n",
      "FLOORSMAX_MODE 0.4976082156410665\n",
      "FLOORSMIN_MODE 0.6784862980511266\n",
      "LANDAREA_MODE 0.5937673774271489\n",
      "LIVINGAPARTMENTS_MODE 0.6835495315614726\n",
      "LIVINGAREA_MODE 0.5019332641759157\n",
      "NONLIVINGAPARTMENTS_MODE 0.6943296337366793\n",
      "NONLIVINGAREA_MODE 0.5517916432257708\n",
      "APARTMENTS_MEDI 0.5074972927797705\n",
      "BASEMENTAREA_MEDI 0.5851595552679416\n",
      "YEARS_BEGINEXPLUATATION_MEDI 0.48781019215572774\n",
      "YEARS_BUILD_MEDI 0.6649778381911542\n",
      "COMMONAREA_MEDI 0.6987229725115525\n",
      "ELEVATORS_MEDI 0.532959796560123\n",
      "ENTRANCES_MEDI 0.503487680115508\n",
      "FLOORSMAX_MEDI 0.4976082156410665\n",
      "FLOORSMIN_MEDI 0.6784862980511266\n",
      "LANDAREA_MEDI 0.5937673774271489\n",
      "LIVINGAPARTMENTS_MEDI 0.6835495315614726\n",
      "LIVINGAREA_MEDI 0.5019332641759157\n",
      "NONLIVINGAPARTMENTS_MEDI 0.6943296337366793\n",
      "NONLIVINGAREA_MEDI 0.5517916432257708\n",
      "FONDKAPREMONT_MODE 0.6838617155158677\n",
      "HOUSETYPE_MODE 0.50176091261776\n",
      "TOTALAREA_MODE 0.4826851722377411\n",
      "WALLSMATERIAL_MODE 0.508407829313423\n",
      "EMERGENCYSTATE_MODE 0.47398304450897694\n"
     ]
    }
   ],
   "source": [
    "#%run NB01-Load.ipynb\n",
    "#%run NB02-EDA-MetaData.ipynb\n",
    "%run NB03-EDA-MetaData-Check.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TARGET dtype:  int64\n",
      "NAME_CONTRACT_TYPE dtype:  object\n",
      "FLAG_OWN_CAR dtype:  object\n",
      "FLAG_OWN_REALTY dtype:  object\n",
      "FLAG_MOBIL dtype:  int64\n",
      "FLAG_EMP_PHONE dtype:  int64\n",
      "FLAG_WORK_PHONE dtype:  int64\n",
      "FLAG_CONT_MOBILE dtype:  int64\n",
      "FLAG_PHONE dtype:  int64\n",
      "FLAG_EMAIL dtype:  int64\n",
      "REG_REGION_NOT_LIVE_REGION dtype:  int64\n",
      "REG_REGION_NOT_WORK_REGION dtype:  int64\n",
      "LIVE_REGION_NOT_WORK_REGION dtype:  int64\n",
      "REG_CITY_NOT_LIVE_CITY dtype:  int64\n",
      "REG_CITY_NOT_WORK_CITY dtype:  int64\n",
      "LIVE_CITY_NOT_WORK_CITY dtype:  int64\n",
      "EMERGENCYSTATE_MODE dtype:  object\n",
      "FLAG_DOCUMENT_2 dtype:  int64\n",
      "FLAG_DOCUMENT_3 dtype:  int64\n",
      "FLAG_DOCUMENT_4 dtype:  int64\n",
      "FLAG_DOCUMENT_5 dtype:  int64\n",
      "FLAG_DOCUMENT_6 dtype:  int64\n",
      "FLAG_DOCUMENT_7 dtype:  int64\n",
      "FLAG_DOCUMENT_8 dtype:  int64\n",
      "FLAG_DOCUMENT_9 dtype:  int64\n",
      "FLAG_DOCUMENT_10 dtype:  int64\n",
      "FLAG_DOCUMENT_11 dtype:  int64\n",
      "FLAG_DOCUMENT_12 dtype:  int64\n",
      "FLAG_DOCUMENT_13 dtype:  int64\n",
      "FLAG_DOCUMENT_14 dtype:  int64\n",
      "FLAG_DOCUMENT_15 dtype:  int64\n",
      "FLAG_DOCUMENT_16 dtype:  int64\n",
      "FLAG_DOCUMENT_17 dtype:  int64\n",
      "FLAG_DOCUMENT_18 dtype:  int64\n",
      "FLAG_DOCUMENT_19 dtype:  int64\n",
      "FLAG_DOCUMENT_20 dtype:  int64\n",
      "FLAG_DOCUMENT_21 dtype:  int64\n"
     ]
    }
   ],
   "source": [
    "# Columns where ‘Count of Unique Values Per Column’ is 2\n",
    "for column_name in df_columns_number_of_unique_values:\n",
    "    if df_columns_number_of_unique_values[column_name] == 2:\n",
    "        print (column_name, 'dtype: ', df[column_name].dtype)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0    282686\n",
      "1     24825\n",
      "Name: TARGET, dtype: int64\n",
      "Cash loans         278232\n",
      "Revolving loans     29279\n",
      "Name: NAME_CONTRACT_TYPE, dtype: int64\n",
      "N    202924\n",
      "Y    104587\n",
      "Name: FLAG_OWN_CAR, dtype: int64\n",
      "Y    213312\n",
      "N     94199\n",
      "Name: FLAG_OWN_REALTY, dtype: int64\n",
      "1    307510\n",
      "0         1\n",
      "Name: FLAG_MOBIL, dtype: int64\n",
      "1    252125\n",
      "0     55386\n",
      "Name: FLAG_EMP_PHONE, dtype: int64\n",
      "0    246203\n",
      "1     61308\n",
      "Name: FLAG_WORK_PHONE, dtype: int64\n",
      "1    306937\n",
      "0       574\n",
      "Name: FLAG_CONT_MOBILE, dtype: int64\n",
      "0    221080\n",
      "1     86431\n",
      "Name: FLAG_PHONE, dtype: int64\n",
      "0    290069\n",
      "1     17442\n",
      "Name: FLAG_EMAIL, dtype: int64\n",
      "0    302854\n",
      "1      4657\n",
      "Name: REG_REGION_NOT_LIVE_REGION, dtype: int64\n",
      "0    291899\n",
      "1     15612\n",
      "Name: REG_REGION_NOT_WORK_REGION, dtype: int64\n",
      "0    295008\n",
      "1     12503\n",
      "Name: LIVE_REGION_NOT_WORK_REGION, dtype: int64\n",
      "0    283472\n",
      "1     24039\n",
      "Name: REG_CITY_NOT_LIVE_CITY, dtype: int64\n",
      "0    236644\n",
      "1     70867\n",
      "Name: REG_CITY_NOT_WORK_CITY, dtype: int64\n",
      "0    252296\n",
      "1     55215\n",
      "Name: LIVE_CITY_NOT_WORK_CITY, dtype: int64\n",
      "No     159428\n",
      "NaN    145755\n",
      "Yes      2328\n",
      "Name: EMERGENCYSTATE_MODE, dtype: int64\n",
      "0    307498\n",
      "1        13\n",
      "Name: FLAG_DOCUMENT_2, dtype: int64\n",
      "1    218340\n",
      "0     89171\n",
      "Name: FLAG_DOCUMENT_3, dtype: int64\n",
      "0    307486\n",
      "1        25\n",
      "Name: FLAG_DOCUMENT_4, dtype: int64\n",
      "0    302863\n",
      "1      4648\n",
      "Name: FLAG_DOCUMENT_5, dtype: int64\n",
      "0    280433\n",
      "1     27078\n",
      "Name: FLAG_DOCUMENT_6, dtype: int64\n",
      "0    307452\n",
      "1        59\n",
      "Name: FLAG_DOCUMENT_7, dtype: int64\n",
      "0    282487\n",
      "1     25024\n",
      "Name: FLAG_DOCUMENT_8, dtype: int64\n",
      "0    306313\n",
      "1      1198\n",
      "Name: FLAG_DOCUMENT_9, dtype: int64\n",
      "0    307504\n",
      "1         7\n",
      "Name: FLAG_DOCUMENT_10, dtype: int64\n",
      "0    306308\n",
      "1      1203\n",
      "Name: FLAG_DOCUMENT_11, dtype: int64\n",
      "0    307509\n",
      "1         2\n",
      "Name: FLAG_DOCUMENT_12, dtype: int64\n",
      "0    306427\n",
      "1      1084\n",
      "Name: FLAG_DOCUMENT_13, dtype: int64\n",
      "0    306608\n",
      "1       903\n",
      "Name: FLAG_DOCUMENT_14, dtype: int64\n",
      "0    307139\n",
      "1       372\n",
      "Name: FLAG_DOCUMENT_15, dtype: int64\n",
      "0    304458\n",
      "1      3053\n",
      "Name: FLAG_DOCUMENT_16, dtype: int64\n",
      "0    307429\n",
      "1        82\n",
      "Name: FLAG_DOCUMENT_17, dtype: int64\n",
      "0    305011\n",
      "1      2500\n",
      "Name: FLAG_DOCUMENT_18, dtype: int64\n",
      "0    307328\n",
      "1       183\n",
      "Name: FLAG_DOCUMENT_19, dtype: int64\n",
      "0    307355\n",
      "1       156\n",
      "Name: FLAG_DOCUMENT_20, dtype: int64\n",
      "0    307408\n",
      "1       103\n",
      "Name: FLAG_DOCUMENT_21, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "# Columns where ‘Count of Unique Values Per Column’ is 2\n",
    "for column_name in df_columns_number_of_unique_values:\n",
    "    if df_columns_number_of_unique_values[column_name] == 2:\n",
    "        print (df[column_name].value_counts(dropna=False))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Columns where ‘Count of Unique Values Per Column’ is 2 (sans NULL)\n",
    "TARGET dtype:  int64  \n",
    "NAME_CONTRACT_TYPE dtype:  object  \n",
    "FLAG_OWN_CAR dtype:  object  \n",
    "FLAG_OWN_REALTY dtype:  object  \n",
    "FLAG_MOBIL dtype:  int64  \n",
    "FLAG_EMP_PHONE dtype:  int64  \n",
    "FLAG_WORK_PHONE dtype:  int64  \n",
    "FLAG_CONT_MOBILE dtype:  int64  \n",
    "FLAG_PHONE dtype:  int64  \n",
    "FLAG_EMAIL dtype:  int64  \n",
    "REG_REGION_NOT_LIVE_REGION dtype:  int64  \n",
    "REG_REGION_NOT_WORK_REGION dtype:  int64  \n",
    "LIVE_REGION_NOT_WORK_REGION dtype:  int64  \n",
    "REG_CITY_NOT_LIVE_CITY dtype:  int64  \n",
    "REG_CITY_NOT_WORK_CITY dtype:  int64  \n",
    "LIVE_CITY_NOT_WORK_CITY dtype:  int64  \n",
    "EMERGENCYSTATE_MODE dtype:  object  \n",
    "FLAG_DOCUMENT_2 dtype:  int64  \n",
    "FLAG_DOCUMENT_3 dtype:  int64  \n",
    "FLAG_DOCUMENT_4 dtype:  int64  \n",
    "FLAG_DOCUMENT_5 dtype:  int64  \n",
    "FLAG_DOCUMENT_6 dtype:  int64  \n",
    "FLAG_DOCUMENT_7 dtype:  int64  \n",
    "FLAG_DOCUMENT_8 dtype:  int64  \n",
    "FLAG_DOCUMENT_9 dtype:  int64  \n",
    "FLAG_DOCUMENT_10 dtype:  int64  \n",
    "FLAG_DOCUMENT_11 dtype:  int64  \n",
    "FLAG_DOCUMENT_12 dtype:  int64  \n",
    "FLAG_DOCUMENT_13 dtype:  int64  \n",
    "FLAG_DOCUMENT_14 dtype:  int64  \n",
    "FLAG_DOCUMENT_15 dtype:  int64  \n",
    "FLAG_DOCUMENT_16 dtype:  int64  \n",
    "FLAG_DOCUMENT_17 dtype:  int64  \n",
    "FLAG_DOCUMENT_18 dtype:  int64  \n",
    "FLAG_DOCUMENT_19 dtype:  int64  \n",
    "FLAG_DOCUMENT_20 dtype:  int64  \n",
    "FLAG_DOCUMENT_21 dtype:  int64  \n",
    "\n",
    "\n",
    "### NOT 0/1 - Binary / Dichotomous / Boolean  \n",
    "\n",
    "\n",
    "Cash loans         278232  \n",
    "Revolving loans     29279  \n",
    "Name: NAME_CONTRACT_TYPE, dtype: int64  \n",
    "\n",
    "\n",
    "N    202924  \n",
    "Y    104587  \n",
    "Name: FLAG_OWN_CAR, dtype: int64  \n",
    "\n",
    "\n",
    "Y    213312  \n",
    "N     94199  \n",
    "Name: FLAG_OWN_REALTY, dtype: int64  \n",
    "\n",
    "\n",
    "No     159428  \n",
    "NaN    145755  \n",
    "Yes      2328  \n",
    "Name: EMERGENCYSTATE_MODE, dtype: int64  \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Transform boolean to 1/0, instead of Y/N, Yes/No/, True/False, etc.  \n",
    "pd.Series(np.where(df.column_name.values == 'yes', 1, 0), df.index)  \n",
    "\n",
    "\n",
    "FLAG_OWN_CAR {N, Y}  \n",
    "\n",
    "\n",
    "FLAG_OWN_REALTY {N, Y}  \n",
    "\n",
    "\n",
    "EMERGENCYSTATE_MODE {No, NaN, Yes}  \n",
    "\n",
    "\n",
    "NAME_CONTRACT_TYPE {Cash loans, Revolving loans}  \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    202924\n",
       "1    104587\n",
       "Name: FLAG_OWN_CAR, dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "temporary_column_name ='FLAG_OWN_CAR'\n",
    "df[temporary_column_name] = pd.Series(np.where(df[temporary_column_name].values == 'Y', 1, 0), df.index)\n",
    "df[temporary_column_name].value_counts(dropna=False, sort=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    213312\n",
       "0     94199\n",
       "Name: FLAG_OWN_REALTY, dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "temporary_column_name ='FLAG_OWN_REALTY'\n",
    "df[temporary_column_name] = pd.Series(np.where(df[temporary_column_name].values == 'Y', 1, 0), df.index)\n",
    "df[temporary_column_name].value_counts(dropna=False, sort=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For the boolean columns, replace TRUE and FALSE with 1 and 0\n",
    "# FLAG_OWN_CAR, FLAG_OWN_REALTY\n",
    "#df['FLAG_OWN_CAR'] = df['FLAG_OWN_CAR'].replace(('Y','N'),(1,0))\n",
    "#df['FLAG_OWN_REALTY'] = df['FLAG_OWN_REALTY'].replace(('Y','N'),(1,0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df.replace(np.nan, -999)\n",
    "#df.fillna(-999)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0         False\n",
       "1         False\n",
       "2         False\n",
       "3         False\n",
       "4         False\n",
       "5         False\n",
       "6         False\n",
       "7         False\n",
       "8          True\n",
       "9         False\n",
       "10        False\n",
       "11         True\n",
       "12        False\n",
       "13        False\n",
       "14        False\n",
       "15        False\n",
       "16        False\n",
       "17        False\n",
       "18        False\n",
       "19        False\n",
       "20        False\n",
       "21        False\n",
       "22        False\n",
       "23         True\n",
       "24        False\n",
       "25        False\n",
       "26        False\n",
       "27        False\n",
       "28        False\n",
       "29        False\n",
       "          ...  \n",
       "307481    False\n",
       "307482    False\n",
       "307483     True\n",
       "307484    False\n",
       "307485    False\n",
       "307486    False\n",
       "307487     True\n",
       "307488    False\n",
       "307489    False\n",
       "307490    False\n",
       "307491    False\n",
       "307492    False\n",
       "307493    False\n",
       "307494    False\n",
       "307495    False\n",
       "307496    False\n",
       "307497    False\n",
       "307498    False\n",
       "307499    False\n",
       "307500    False\n",
       "307501    False\n",
       "307502    False\n",
       "307503    False\n",
       "307504    False\n",
       "307505     True\n",
       "307506    False\n",
       "307507     True\n",
       "307508    False\n",
       "307509    False\n",
       "307510    False\n",
       "Length: 307511, dtype: bool"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.isin(['XNA']).any(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    278232\n",
       "1     29279\n",
       "Name: NAME_CONTRACT_TYPE__Revolving_loans, dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#df['NAME_CONTRACT_TYPE__Revolving_loans'] = pd.Series(np.where(df['NAME_CONTRACT_TYPE'].values == 'Revolving loans', 1, 0), df.index)\n",
    "temporary_column_name = 'NAME_CONTRACT_TYPE'\n",
    "df['NAME_CONTRACT_TYPE__Revolving_loans'] = pd.Series(np.where(df[temporary_column_name].values == 'Revolving loans', 1, 0), df.index)\n",
    "df['NAME_CONTRACT_TYPE__Revolving_loans'].value_counts(dropna=False, sort=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "No         159428\n",
       "MISSING    145755\n",
       "Yes          2328\n",
       "Name: EMERGENCYSTATE_MODE, dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#df['EMERGENCYSTATE_MODE'].fillna('MISSING', inplace=True)\n",
    "temporary_column_name ='EMERGENCYSTATE_MODE'\n",
    "df[temporary_column_name].fillna('MISSING', inplace=True)\n",
    "df[temporary_column_name].value_counts(dropna=False, sort=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    161756\n",
       "1    145755\n",
       "Name: EMERGENCYSTATE_MODE__MISSING, dtype: int64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#df['EMERGENCYSTATE_MODE__MISSING'] = pd.Series(np.where(df['EMERGENCYSTATE_MODE'].values == 'MISSING', 1, 0), df.index)\n",
    "temporary_column_name = 'EMERGENCYSTATE_MODE'\n",
    "df['EMERGENCYSTATE_MODE__MISSING'] = pd.Series(np.where(df[temporary_column_name].values == 'MISSING', 1, 0), df.index)\n",
    "df['EMERGENCYSTATE_MODE__MISSING'].value_counts(dropna=False, sort=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    305183\n",
       "1      2328\n",
       "Name: EMERGENCYSTATE_MODE__Yes, dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#df['EMERGENCYSTATE_MODE__Yes'] = pd.Series(np.where(df['EMERGENCYSTATE_MODE'].values == 'Yes', 1, 0), df.index)\n",
    "temporary_column_name = 'EMERGENCYSTATE_MODE'\n",
    "df['EMERGENCYSTATE_MODE__Yes'] = pd.Series(np.where(df[temporary_column_name].values == 'Yes', 1, 0), df.index)\n",
    "df['EMERGENCYSTATE_MODE__Yes'].value_counts(dropna=False, sort=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['FLAG_MOBIL'].value_counts(dropna=False)\n",
    "df['FLAG_EMP_PHONE'].value_counts(dropna=False)\n",
    "df['FLAG_WORK_PHONE'].value_counts(dropna=False)\n",
    "df['FLAG_CONT_MOBILE'].value_counts(dropna=False)\n",
    "df['FLAG_PHONE'].value_counts(dropna=False)\n",
    "df['FLAG_EMAIL'].value_counts(dropna=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# List of unique values in the df['name'] column\n",
    "# df.name.unique()\n",
    "df.TARGET.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['TARGET'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Number of unique values in the df['name'] column\n",
    "df.TARGET.nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['TARGET'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['TARGET'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://stackoverflow.com/questions/45759966/counting-unique-values-in-a-column-in-pandas-dataframe-like-in-qlik\n",
    "\n",
    "Count distict values, use nunique:\n",
    "df['hID'].nunique()\n",
    "Count only non-null values, use count:\n",
    "df['hID'].count()\n",
    "Count total values including null values, use size attribute:\n",
    "df['hID'].size\n",
    "To add condition...\n",
    "Use boolean indexing:\n",
    "df.loc[df['mID']=='A','hID'].agg(['nunique','count','size'])\n",
    "Or, using query:\n",
    "df.query('mID == \"A\"')['hID'].agg(['nunique','count','size'])\n",
    "\n",
    "New in pandas 0.20.0 pd.DataFrame.agg\n",
    "df.agg(['count', 'size', 'nunique'])\n",
    "You've always been able to do an agg within a groupby. I used stack at the end because I like the presentation better.\n",
    "df.groupby('mID').agg(['count', 'size', 'nunique']).stack()\n",
    "\n",
    "\n",
    "https://stackoverflow.com/questions/45125408/how-to-count-the-distinct-values-across-a-column-in-pandas\n",
    "\n",
    "df[['Company', 'Date']].drop_duplicates()['Company'].value_counts()\n",
    "df.groupby('Company')['Date'].nunique()\n",
    "\n",
    "\n",
    "https://stackoverflow.com/questions/48162201/pandas-number-of-unique-values-and-sort-by-the-number-of-unique\n",
    "\n",
    "df = df.groupby('A')['B'].nunique().sort_values(ascending=False).reset_index(name='count')\n",
    "print (df)\n",
    "\n",
    "\n",
    "https://stackoverflow.com/questions/38309729/count-unique-values-with-pandas-per-groups/38309823\n",
    "\n",
    "You can retain the column name like this:\n",
    "df = df.groupby(by='domain', as_index=False).agg({'ID': pd.Series.nunique})\n",
    "The difference is that 'nunique()' returns a Series and 'agg()' returns a DataFrame.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(df.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(df.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.shape[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_row_count, df_column_count = df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_row_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_column_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dict((column_name,None) for column_name in df_column_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dict((column_name, df_row_count) for column_name in df_column_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://stackoverflow.com/questions/3869487/how-do-i-create-a-dictionary-with-keys-from-a-list-and-values-defaulting-to-say\n",
    "# Generator expressions avoid the memory overhead of populating the whole list.\n",
    "# https://stackoverflow.com/questions/2241891/how-to-initialize-a-dict-with-keys-from-a-list-and-empty-value-in-python\n",
    "# dict.fromkeys(keys_list)\n",
    "# Be careful with initializing to something mutable: If you call, e.g., dict.fromkeys([1, 2, 3], []), all of the keys are mapped to the same list, and modifying one will modify them all.\n",
    "# \n",
    "# dict-comprehension solution\n",
    "# keys = [1,2,3,5,6,7]\n",
    "# {key: None for key in keys}\n",
    "#> {1: None, 2: None, 3: None, 5: None, 6: None, 7: None}\n",
    "# \n",
    "# Using a dict-comp also allows the value to be the result of calling a function\n",
    "#   (which could be passed the key as an argument, if desired)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "columns_row_count = {column_name:df_row_count for column_name in df_column_names}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "columns_row_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "columns_number_of_unique_values = {column_name:None for column_name in df_column_names}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "columns_number_of_unique_values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for column_name in columns_number_of_unique_values:\n",
    "    columns_number_of_unique_values[column_name] = df[column_name].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "columns_number_of_unique_values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "columns_cardinality = {column_name:None for column_name in df_column_names}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "columns_cardinality"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for column_name in columns_cardinality:\n",
    "    columns_cardinality[column_name] = columns_number_of_unique_values[column_name]/df_row_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "columns_cardinality"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for column_name in columns_number_of_unique_values:\n",
    "    if columns_number_of_unique_values[column_name] == 2:\n",
    "        print (column_name, 'dtype: ', df[column_name].dtype)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "# http://pbpython.com/pandas-list-dict.html\n",
    "The “default” manner to create a DataFrame from python is to use a list of dictionaries.\n",
    "In this case each dictionary key is used for the column headings.\n",
    "A default index will be created automatically:\n",
    "sales = [{'account': 'Jones LLC', 'Jan': 150, 'Feb': 200, 'Mar': 140},\n",
    "         {'account': 'Alpha Co',  'Jan': 200, 'Feb': 210, 'Mar': 215},\n",
    "         {'account': 'Blue Inc',  'Jan': 50,  'Feb': 90,  'Mar': 95 }]\n",
    "df = pd.DataFrame(sales)\n",
    "\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "empty_dictionary = {}\n",
    "empty_dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "any(empty_dictionary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(empty_dictionary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bool(empty_dictionary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "falsy_dictionary = {0:False}\n",
    "falsy_dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "any(falsy_dictionary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(falsy_dictionary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bool(falsy_dictionary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "# https://stackoverflow.com/questions/23177439/python-checking-if-a-dictionary-is-empty-doesnt-seem-to-work\n",
    "test_dict = {}\n",
    "\n",
    "# Option 1\n",
    "if not test_dict:\n",
    "    print \"Dict is Empty\"\n",
    "\n",
    "# Option 2\n",
    "if not bool(test_dict):\n",
    "    print \"Dict is Empty\"\n",
    "\n",
    "# Option 3\n",
    "if len(test_dict) == 0:\n",
    "    print \"Dict is Empty\"\n",
    "\n",
    "# The first test in the answer above is true not only if the dict exists and is empty, but also if test_dict is None.\n",
    "# So use this test only when you know that the dict object exists (or when the difference does not matter).\n",
    "# The second way also has that behavior.\n",
    "# Only the third way barks if test_dict is None.\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://stackoverflow.com/questions/18667410/how-can-i-check-if-a-string-only-contains-letters-in-python\n",
    "\n",
    "if string.isalpha():\n",
    "    print(\"It's all letters\")\n",
    "\n",
    "# NOTE: using [A-Za-z ]+ will not match names with non ASCII letterss\n",
    "# NOTE: using \\w includes digits\n",
    "import re\n",
    "def only_letters(tested_string):\n",
    "    #match = re.match(\"^[ABCDEFGHJKLM]*$\", tested_string)\n",
    "    #match = re.match(\"^[A-HJ-M]*$\", tested_string)\n",
    "    match = re.match(\"^[A-Za-z]*$\", tested_string)\n",
    "    return match is not None\n",
    "\n",
    "import re\n",
    "def only_letters(string):\n",
    "    return re.match(r'[a-z\\s]+$',string,2) # JWB: What's with the \",2\"?\n",
    "\n",
    "def only_letters(string):\n",
    "    return all(letter.isalpha() for letter in string)\n",
    "\n",
    "def only_letters(s):\n",
    "    for c in s:\n",
    "        cat = unicodedata.category(c)\n",
    "        # Lu: Category: Letter, Uppercase (https://codepoints.net/search?gc=Lu)\n",
    "        # Ll: Category: Letter, Lowercase (https://codepoints.net/search?gc=Ll)\n",
    "        # Lt: Category: Letter, Titlecase (https://codepoints.net/search?gc=Lt)\n",
    "        # Lm: Category: Letter, Modifier  (https://codepoints.net/search?gc=Lm)\n",
    "        # Lo: Category: Letter, Other     (https://codepoints.net/search?gc=Lo)\n",
    "        # Latin-1: There are no Lm or Lt category codepoints in the Latin-1 subset of Unicode and only 2 Lo characters, ª (U+00AA) and º (U+00BA), Feminine and Masculine Ordinal Indicator).\n",
    "        if cat not in ('Lu','Ll','Lo'):\n",
    "            return False\n",
    "    return True\n",
    "\n",
    "# https://stackoverflow.com/questions/29460405/checking-if-string-is-only-letters-and-spaces-python\n",
    "# To require that the string contains only alphas and spaces:\n",
    "if all(x.isalpha() or x.isspace() for x in string):\n",
    "    print(\"Only alphabetical letters and spaces: yes\")\n",
    "else:\n",
    "    print(\"Only alphabetical letters and spaces: no\")\n",
    "# To require that the string contains at least one alpha and at least one space:\n",
    "if any(x.isalpha() for x in string) and any(x.isspace() for x in string):\n",
    "# To require that the string contains at least one alpha, at least one space, and only alphas and spaces:\n",
    "if (any(x.isalpha() for x in string)\n",
    "    and any(x.isspace() for x in string)\n",
    "    and all(x.isalpha() or x.isspace() for x in string)):\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#locale.getlocale()\n",
    "#> NameError: name 'locale' is not defined"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'dog'.isalpha()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'äöå'.isalpha()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'привіт'.isalpha()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "u'привіт'.isalpha()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "repr('äöå')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ascii('äöå')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://stackoverflow.com/questions/4286637/python-isalpha-and-scandics\n",
    "# http://en.wikipedia.org/wiki/Windows-1252\n",
    "s = '\\xe4\\xf6\\xe5'\n",
    "import unicodedata\n",
    "for c in s:\n",
    "    u = c.decode('1252')\n",
    "    print (ascii(c), ascii(u), unicodedata.name(u, '<no name>'))\n",
    "#'\\xe4' u'\\xe4' LATIN SMALL LETTER A WITH DIAERESIS\n",
    "#'\\xf6' u'\\xf6' LATIN SMALL LETTER O WITH DIAERESIS\n",
    "#'\\xe5' u'\\xe5' LATIN SMALL LETTER A WITH RING ABOVE\n",
    "s.isalpha()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The isalpha() methods returns “True” if all characters in the string are alphabets, Otherwise, It returns “False”.\n",
    "# This function is used to check if the argument contains any alphabets characters such as:\n",
    "#     ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz\n",
    "# Both uppercase and lowercase alphabets return “True”\n",
    "# Space is not considered to be alphabet, therefore it returns “False”\n",
    "\n",
    "def is_plain_text(string):\n",
    "    if string != 0:\n",
    "            # require that the string contains only alphas\n",
    "            #if all(character.isalpha() for character in string)\n",
    "            # require that the string contains only alphas and spaces\n",
    "            if all(character.isalpha() or character.isspace() for character in string):\n",
    "                return True\n",
    "            else:\n",
    "                return False\n",
    "    else:\n",
    "            return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "is_plain_text(df_as_objects['SK_ID_CURR'])\n",
    "#TypeError: 'numpy.bool_' object is not iterable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dtypes = df.dtypes\n",
    "dtypes = dtypes[dtypes != 'object']\n",
    "features = list(set(dtypes.index) - set(['TARGET']))\n",
    "len(features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = df[features]\n",
    "y = df['TARGET']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "id_column = ['SK_ID_CURR']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "target_column = ['TARGET']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "boolean_columns = ['FLAG_OWN_CAR', 'FLAG_OWN_REALTY', 'FLAG_MOBIL', 'FLAG_EMP_PHONE', 'FLAG_WORK_PHONE', 'FLAG_CONT_MOBILE', 'FLAG_PHONE', 'FLAG_EMAIL', 'REG_REGION_NOT_LIVE_REGION', 'REG_REGION_NOT_WORK_REGION', 'LIVE_REGION_NOT_WORK_REGION', 'REG_CITY_NOT_LIVE_CITY', 'REG_CITY_NOT_WORK_CITY', 'LIVE_CITY_NOT_WORK_CITY', 'FLAG_DOCUMENT_2', 'FLAG_DOCUMENT_3', 'FLAG_DOCUMENT_4', 'FLAG_DOCUMENT_5', 'FLAG_DOCUMENT_6', 'FLAG_DOCUMENT_7', 'FLAG_DOCUMENT_8', 'FLAG_DOCUMENT_9', 'FLAG_DOCUMENT_10', 'FLAG_DOCUMENT_11', 'FLAG_DOCUMENT_12', 'FLAG_DOCUMENT_13', 'FLAG_DOCUMENT_14', 'FLAG_DOCUMENT_15', 'FLAG_DOCUMENT_16', 'FLAG_DOCUMENT_17', 'FLAG_DOCUMENT_18', 'FLAG_DOCUMENT_19', 'FLAG_DOCUMENT_20', 'FLAG_DOCUMENT_21']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "categorical_columns = ['NAME_CONTRACT_TYPE', 'CODE_GENDER', 'NAME_TYPE_SUITE', 'NAME_INCOME_TYPE', 'NAME_EDUCATION_TYPE', 'NAME_FAMILY_STATUS', 'NAME_HOUSING_TYPE', 'OCCUPATION_TYPE', 'WEEKDAY_APPR_PROCESS_START', 'ORGANIZATION_TYPE', 'FONDKAPREMONT_MODE', 'HOUSETYPE_MODE', 'WALLSMATERIAL_MODE', 'EMERGENCYSTATE_MODE']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "object_columns = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "numerical_columns = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "int_columns = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "float_columns = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "datetime_columns = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "date_columns = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nondata_columns = ['SK_ID_CURR']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "target_distribution = df['TARGET'].value_counts()\n",
    "target_distribution.plot.pie(figsize=(10, 10),\n",
    "                             title='Target Distribution',\n",
    "                             fontsize=15, \n",
    "                             legend=True,\n",
    "                             autopct=lambda v: \"{:0.1f}%\".format(v))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "total_nans = df.isnull().sum()\n",
    "total_nans"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nan_precents = (df.isnull().sum()/df.isnull().count()*100)\n",
    "feature_overview_df  = pd.concat([total_nans, nan_precents], axis=1, keys=['NaN Count', 'NaN Pencent'])\n",
    "feature_overview_df['Type'] = [application_train[c].dtype for c in feature_overview_df.index]\n",
    "pd.set_option('display.max_rows', None)\n",
    "display(feature_overview_df)\n",
    "pd.set_option('display.max_rows', 20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_application_is_nan_df = pd.DataFrame()\n",
    "for column in df.columns:\n",
    "    if application_train[column].isnull().sum() == 0:\n",
    "        continue\n",
    "    all_application_is_nan_df['is_nan_' + column] = df[column].isnull()\n",
    "    all_application_is_nan_df['is_nan_' + column] = all_application_is_nan_df['is_nan_' + column].map(lambda v: 1 if v else 0)\n",
    "all_application_is_nan_df['target'] = df['TARGET']\n",
    "all_application_is_nan_df = all_application_is_nan_df[pd.notnull(all_application_is_nan_df['target'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "display(all_application_is_nan_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y = all_application_is_nan_df.pop('target')\n",
    "X = all_application_is_nan_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X, valid_X, train_Y, valid_Y = train_test_split(X, Y, test_size=0.2, random_state=2018)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf = LGBMClassifier(n_estimators=200, learning_rate=0.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf.fit(\n",
    "        train_X,\n",
    "        train_Y,\n",
    "        eval_set=[(train_X, train_Y), (valid_X, valid_Y)],\n",
    "        eval_metric='auc',\n",
    "        early_stopping_rounds=50,\n",
    "        verbose=False\n",
    "       )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_importance(clf, figsize=(10,10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(\"only showing the distribution for the first few columns, edit the counter to show all distribution\")\n",
    "#show_feature_count = 10\n",
    "#for column in all_application_df.columns:\n",
    "#   if show_feature_count == 0:\n",
    "#        break\n",
    "#    show_feature_count -= 1\n",
    "#    draw_feature_distribution(all_application_df, column)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "draw_feature_distribution(df, 'TARGET')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "draw_feature_distribution(df, 'DAYS_EMPLOYED')\n",
    "# ToDo(JamesBalcomb): fix \"ValueError: max() arg is an empty sequence\" - add check for 'class_t_values'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#EXT_SOURCE_1\n",
    "#EXT_SOURCE_2\n",
    "#EXT_SOURCE_3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "draw_feature_distribution(df, 'EXT_SOURCE_1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "draw_feature_distribution(df, 'EXT_SOURCE_2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "draw_feature_distribution(df, 'EXT_SOURCE_3')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# EXT_SOURCE_1, EXT_SOURCE_2, EXT_SOURCE_3\n",
    "\n",
    "Q: Is there a relationship between any of these three continuous variables and the binary classification target variable?\n",
    "A: Yes, but EXT_SOURCE_2 is oddly shapen."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Seaborn Violin Plot - correlation; distribution and density\n",
    "sns.violinplot(x='TARGET', y='EXT_SOURCE_1', data=df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Seaborn Violin Plot - correlation; distribution and density\n",
    "sns.violinplot(x='TARGET', y='EXT_SOURCE_2', data=df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Seaborn Violin Plot - correlation; distribution and density\n",
    "sns.violinplot(x='TARGET', y='EXT_SOURCE_3', data=df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfapplication_train.hist(column='EXT_SOURCE_1', # Column to plot\n",
    "              figsize=(8,8),                  # Plot size\n",
    "              color=\"blue\"                    # Plot color\n",
    "              )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.hist(column='EXT_SOURCE_2', # Column to plot\n",
    "              figsize=(8,8),                  # Plot size\n",
    "              color=\"blue\"                    # Plot color\n",
    "              )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.hist(column='EXT_SOURCE_3', # Column to plot\n",
    "              figsize=(8,8),                  # Plot size\n",
    "              color=\"blue\"                    # Plot color\n",
    "              )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://gist.github.com/ltfschoen/4c5d2cf26b8be5355043273493a6b8b9#file-proportions_of_missing_data_in_dataframe_columns-py\n",
    "def get_percentage_missing(series):\n",
    "    \"\"\" Calculates percentage of NaN values in DataFrame\n",
    "    :param series: Pandas DataFrame object\n",
    "    :return: float\n",
    "    \"\"\"\n",
    "    num = series.isnull().sum()\n",
    "    den = len(series)\n",
    "    return round(num / den, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "get_percentage_missing(df['EXT_SOURCE_1'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "get_percentage_missing(df['EXT_SOURCE_2'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "get_percentage_missing(df['EXT_SOURCE_3'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # https://datascience.stackexchange.com/questions/12645/how-to-count-the-number-of-missing-values-in-each-row-in-pandas-dataframe\n",
    "# # Count of Missing Values per Column\n",
    "# df.isnull().sum(axis=0)\n",
    "# # Count of Missing Values per Row\n",
    "# df.isnull().sum(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://towardsdatascience.com/the-tale-of-missing-values-in-python-c96beb0e8a9d\n",
    "# If the missing value isn’t identified as NaN , then we have to first convert or replace such non NaN entry with a NaN.\n",
    "data_name[‘column_name’].replace(0, np.nan, inplace= True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://gist.github.com/ltfschoen/4c5d2cf26b8be5355043273493a6b8b9#file-proportions_of_missing_data_in_dataframe_columns-py\n",
    "df = application_train\n",
    "# Only include columns that contain any NaN values\n",
    "df_with_any_null_values = df[df.columns[df.isnull().any()].tolist()]\n",
    "\n",
    "get_percentage_missing(df_with_any_null_values)\n",
    "\n",
    "# Iterate over columns in DataFrame and delete those with where >30% of the values are null/NaN\n",
    "for name, values in df_with_any_null_values.iteritems():\n",
    "    # print(\"%r: %r\" % (name, values))\n",
    "    if get_percentage_missing(df_with_any_null_values[name]) > 0.30:\n",
    "        print(\"Deleting Column %r: \" % (name))\n",
    "        # df_with_any_null_values.drop(name, axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(12,5))\n",
    "plt.title(\"Distribution of EXT_SOURCE_1\")\n",
    "ax = sns.distplot(application_traindf[\"EXT_SOURCE_1\"].dropna())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(12,5))\n",
    "plt.title(\"Distribution of EXT_SOURCE_2\")\n",
    "ax = sns.distplot(df[\"EXT_SOURCE_2\"].dropna())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(12,5))\n",
    "plt.title(\"Distribution of EXT_SOURCE_3\")\n",
    "ax = sns.distplot(df[\"EXT_SOURCE_3\"].dropna())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['EXT_SOURCE_AVG'] = df[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']].mean(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(12,5))\n",
    "plt.title(\"Distribution of EXT_SOURCE_AVG\")\n",
    "ax = sns.distplot(df[\"EXT_SOURCE_AVG\"].dropna())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://stackoverflow.com/questions/35277075/python-pandas-counting-the-occurrences-of-a-specific-value\n",
    "#df.loc[df.education == '9th', 'education'].count()\n",
    "#(df.education == '9th').sum()\n",
    "#df.query('education == \"9th\"').education.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.loc[df.EXT_SOURCE_1 == 0.0, 'EXT_SOURCE_1'].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.loc[df.EXT_SOURCE_2 == 0.0, 'EXT_SOURCE_2'].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.loc[df.EXT_SOURCE_3 == 0.0, 'EXT_SOURCE_3'].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['EXT_SOURCE_1'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['EXT_SOURCE_2'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['EXT_SOURCE_3'].value_counts()\n",
    "# df['EXT_SOURCE_3'].value_counts()[:20]\n",
    "# # ValueError: index must be monotonic increasing or decreasing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['EXT_SOURCE_1'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['EXT_SOURCE_2'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['EXT_SOURCE_3'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['EXT_SOURCE_1'].plot(kind='hist', bins=[0.0,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1.0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['EXT_SOURCE_1'].plot(kind='hist', bins=[0.0,0.2,0.4,0.6,0.8,1.0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['EXT_SOURCE_2'].plot(kind='hist', bins=[0.0,0.2,0.4,0.6,0.8,1.0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['EXT_SOURCE_3'].plot(kind='hist', bins=[0.0,0.2,0.4,0.6,0.8,1.0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://community.modeanalytics.com/python/tutorial/python-histograms-boxplots-and-distributions/\n",
    "bin_values = np.arange(start=0, stop=1, step=0.2)\n",
    "us_mq_airlines_index = df['TARGET'].isin(['US','MQ']) # create index\n",
    "us_mq_airlines = df[us_mq_airlines_index] # select rows\n",
    "group_carriers = us_mq_airlines.groupby('TARGET')['EXT_SOURCE_1'] # group values\n",
    "group_carriers.plot(kind='hist', bins=bin_values, figsize=[12,6], alpha=.4, legend=True) # alpha for transparency"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['EXT_SOURCE_1'].plot(kind='box', figsize=[16,8])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['EXT_SOURCE_2'].plot(kind='box', figsize=[16,8])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['EXT_SOURCE_3'].plot(kind='box', figsize=[16,8])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define a function to create the scatterplot. This makes it easy to\n",
    "# reuse code within and across notebooks\n",
    "def scatterplot(x_data, y_data, x_label, y_label, title):\n",
    "\n",
    "    # Create the plot object\n",
    "    _, ax = plt.subplots()\n",
    "\n",
    "    # Plot the data, set the size (s), color and transparency (alpha) of the points\n",
    "    ax.scatter(x_data, y_data, s = 30, color = '#539caf', alpha = 0.75)\n",
    "\n",
    "    # Label the axes and provide a title\n",
    "    ax.set_title(title)\n",
    "    ax.set_xlabel(x_label)\n",
    "    ax.set_ylabel(y_label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Call the function to create plot\n",
    "#scatterplot(x_data = daily_data['temp']\n",
    "#            , y_data = daily_data['cnt']\n",
    "#            , x_label = 'Normalized temperature (C)'\n",
    "#            , y_label = 'Check outs'\n",
    "#            , title = 'Number of Check Outs vs Temperature')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Call the function to create plot\n",
    "scatterplot(x_data = df['EXT_SOURCE_1']\n",
    "            , y_data = df['EXT_SOURCE_2']\n",
    "            , x_label = 'EXT_SOURCE_1'\n",
    "            , y_label = 'EXT_SOURCE_2'\n",
    "            , title = 'EXT_SOURCE_1 vs. EXT_SOURCE_2'\n",
    "           )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Call the function to create plot\n",
    "scatterplot(x_data = df['EXT_SOURCE_1']\n",
    "            , y_data = df['EXT_SOURCE_3']\n",
    "            , x_label = 'EXT_SOURCE_1'\n",
    "            , y_label = 'EXT_SOURCE_2'\n",
    "            , title = 'EXT_SOURCE_1 vs. EXT_SOURCE_3'\n",
    "           )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Call the function to create plot\n",
    "scatterplot(x_data = df['EXT_SOURCE_2']\n",
    "            , y_data = df['EXT_SOURCE_3']\n",
    "            , x_label = 'EXT_SOURCE_2'\n",
    "            , y_label = 'EXT_SOURCE_3'\n",
    "            , title = 'EXT_SOURCE_2 vs. EXT_SOURCE_3'\n",
    "           )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ext_source_1__dropna = df.loc[:,['EXT_SOURCE_1','TARGET']]\n",
    "df_ext_source_1__dropna.dropna(inplace = True)\n",
    "scipy.stats.pointbiserialr(df_ext_source_1__dropna['EXT_SOURCE_1'], df_ext_source_1__dropna['TARGET'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ext_source_2__dropna = application_train.loc[:,['EXT_SOURCE_2','TARGET']]\n",
    "df_ext_source_2__dropna.dropna(inplace = True)\n",
    "scipy.stats.pointbiserialr(df_ext_source_2__dropna['EXT_SOURCE_2'], df_ext_source_2__dropna['TARGET'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ext_source_3__dropna = df.loc[:,['EXT_SOURCE_3','TARGET']]\n",
    "df_ext_source_3__dropna.dropna(inplace = True)\n",
    "scipy.stats.pointbiserialr(df_ext_source_3__dropna['EXT_SOURCE_3'], df_ext_source_3__dropna['TARGET'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scipy.stats.pearsonr(df_ext_source_1__dropna['EXT_SOURCE_1'], df_ext_source_1__dropna['TARGET'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scipy.stats.pearsonr(df_ext_source_2__dropna['EXT_SOURCE_2'], df_ext_source_2__dropna['TARGET'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scipy.stats.pearsonr(df_ext_source_3__dropna['EXT_SOURCE_3'], df_ext_source_3__dropna['TARGET'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.corrcoef(df_ext_source_1__dropna['EXT_SOURCE_1'], df_ext_source_1__dropna['TARGET'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.corrcoef(df_ext_source_2__dropna['EXT_SOURCE_2'], df_ext_source_2__dropna['TARGET'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.corrcoef(df_ext_source_3__dropna['EXT_SOURCE_3'], df_ext_source_3__dropna['TARGET'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp = previous_application[\"NAME_CONTRACT_TYPE\"].value_counts()\n",
    "fig = {\n",
    "  \"data\": [\n",
    "    {\n",
    "      \"values\": temp.values,\n",
    "      \"labels\": temp.index,\n",
    "      \"domain\": {\"x\": [0, .48]},\n",
    "      #\"name\": \"Types of Loans\",\n",
    "      #\"hoverinfo\":\"label+percent+name\",\n",
    "      \"hole\": .7,\n",
    "      \"type\": \"pie\"\n",
    "    },\n",
    "    \n",
    "    ],\n",
    "  \"layout\": {\n",
    "        \"title\":\"Contract product type of previous application\",\n",
    "        \"annotations\": [\n",
    "            {\n",
    "                \"font\": {\n",
    "                    \"size\": 20\n",
    "                },\n",
    "                \"showarrow\": False,\n",
    "                \"text\": \"Contract product type\",\n",
    "                \"x\": 0.12,\n",
    "                \"y\": 0.5\n",
    "            }\n",
    "            \n",
    "        ]\n",
    "    }\n",
    "}\n",
    "iplot(fig, filename='donut')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pearson Correlation of features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = [\n",
    "    go.Heatmap(\n",
    "        z=df.corr().values,\n",
    "        x=df.columns.values,\n",
    "        y=df.columns.values,\n",
    "        colorscale='Viridis',\n",
    "        reversescale = False,\n",
    "        text = True ,\n",
    "        opacity = 1.0 )\n",
    "]\n",
    "\n",
    "layout = go.Layout(\n",
    "    title='Pearson Correlation of features',\n",
    "    xaxis = dict(ticks='', nticks=36),\n",
    "    yaxis = dict(ticks='' ),\n",
    "    width = 900, height = 700,\n",
    "margin=dict(\n",
    "    l=240,\n",
    "),)\n",
    "\n",
    "fig = go.Figure(data=data, layout=layout)\n",
    "py.iplot(fig, filename='labelled-heatmap')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Multicollinearity\n",
    "https://stackoverflow.com/questions/25676145/capturing-high-multi-collinearity-in-statsmodels/44012251#44012251\n",
    "https://stackoverflow.com/questions/25676145/capturing-high-multi-collinearity-in-statsmodels/25833792#25833792\n",
    "https://onlinecourses.science.psu.edu/stat501/node/347/\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## https://stackoverflow.com/questions/25676145/capturing-high-multi-collinearity-in-statsmodels/44012251#44012251\n",
    "##...looking for a single number that captured the collinearity\n",
    "##...options include the determinant and condition number of the correlation matrix\n",
    "##...determinant of the correlation matrix will \"range from 0 (Perfect Collinearity) to 1 (No Collinearity)\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compute correlation matrices\n",
    "pearson_product_moment_correlation_coefficients = np.corrcoef(df, rowvar=0)\n",
    "## https://docs.scipy.org/doc/numpy/reference/generated/numpy.corrcoef.html\n",
    "## Return Pearson product-moment correlation coefficients."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compare the determinants\n",
    "print np.linalg.det(pearson_product_moment_correlation_coefficients)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# the condition number of the covariance matrix will approach infinity with perfect linear dependence\n",
    "print np.linalg.cond(pearson_product_moment_correlation_coefficients)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://towardsdatascience.com/automated-feature-engineering-in-python-99baf11cc219\n",
    "# Group loans by client id and calculate mean, max, min of loans\n",
    "stats = loans.groupby('client_id')['loan_amount'].agg(['mean', 'max', 'min'])\n",
    "stats.columns = ['mean_loan_amount', 'max_loan_amount', 'min_loan_amount']\n",
    "\n",
    "# Merge with the clients dataframe\n",
    "stats = clients.merge(stats, left_on = 'client_id', right_index=True, how = 'left')\n",
    "\n",
    "stats.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#F      202448\n",
    "#M      105059\n",
    "#XNA         4\n",
    "#Name: CODE_GENDER, dtype: int64\n",
    "202448 / (202448 + 105059), 105059 / (202448 + 105059)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#F      202448\n",
    "#M      105059\n",
    "#XNA         4\n",
    "#Name: CODE_GENDER, dtype: int64\n",
    "(4 * 0.658352492788782), (4 * 0.34164750721121795)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Female: 202448 (0.6584) Male: 105059 (0.3417)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# application_train[application_train['CODE_GENDER'] == \"XNA\"]\n",
    "#        SK_ID_CURR\n",
    "#  35657     141289\n",
    "#  38566     144669\n",
    "#  83382     196708\n",
    "# 189640     319880\n",
    "temporary_list = df[df['CODE_GENDER'] == \"XNA\"]['SK_ID_CURR'].tolist()\n",
    "temporary_list\n",
    "#> [141289, 144669, 196708, 319880]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "# https://www.safaribooksonline.com/library/view/python-cookbook/0596001673/ch02s09.html\n",
    "# But the winner is the version that appears to be the simplest:\n",
    "def best(  ):\n",
    "    random.shuffle(data)\n",
    "    for elem in data: process(elem)\n",
    "# Or, if you need to preserve the data list's original ordering:\n",
    "def best_preserve(  ):\n",
    "    aux = list(data)\n",
    "    random.shuffle(aux)\n",
    "    for elem in aux: process(elem)\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "# https://www.pythoncentral.io/select-random-item-list-tuple-data-structure-python/\n",
    "# One of the most common tasks that requires random action is selecting one item from a group, be it a character from a string, unicode, or buffer, a byte from a bytearray, or an item from a list, tuple, set, or xrange.\n",
    "# It's also common to want a sample of more than one item.\n",
    "# The pythonic way to select a single item from a Python sequence type — that's any of str, unicode, list, tuple, bytearray, buffer, xrange — is to use random.choice.\n",
    "# For example, the last line of our single-item selection would be:\n",
    "rand_item = random.choice(items)\n",
    "# Much simpler, isn't it? There's an equally simple way to select n items from the sequence:\n",
    "rand_items = random.sample(items, n)\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#set seed = 1234567890\n",
    "random_item = random.choice(temporary_list)\n",
    "random_item\n",
    "#> 196708"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "random_items = random.sample(temporary_list, 3)\n",
    "random_items\n",
    "#> [141289, 319880, 196708]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Python 3.6+\n",
    "from random import choices\n",
    "random_items = choices(temporary_list, k=3)\n",
    "random_items\n",
    "#> [141289, 144669, 141289]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# CODE_GENDER: XNA ==> CODE_GENDER: F\n",
    "#application_train[application_train['SK_ID_CURR'] == 141289]\n",
    "df[df['SK_ID_CURR'] == 141289]['CODE_GENDER']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.loc[df.SK_ID_CURR == 141289, 'CODE_GENDER'] = 'F'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[df['SK_ID_CURR'] == 141289]['CODE_GENDER']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# CODE_GENDER: XNA ==> CODE_GENDER: F\n",
    "df[df['SK_ID_CURR'] == 319880]['CODE_GENDER']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.loc[df.SK_ID_CURR == 319880, 'CODE_GENDER'] = 'F'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[df['SK_ID_CURR'] == 319880]['CODE_GENDER']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# CODE_GENDER: XNA ==> CODE_GENDER: F\n",
    "df[df['SK_ID_CURR'] == 196708]['CODE_GENDER']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.loc[df.SK_ID_CURR == 196708, 'CODE_GENDER'] = 'F'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[df['SK_ID_CURR'] == 196708]['CODE_GENDER']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# CODE_GENDER: XNA ==> CODE_GENDER: M\n",
    "df[df['SK_ID_CURR'] == 144669]['CODE_GENDER']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.loc[df.SK_ID_CURR == 144669, 'CODE_GENDER'] = 'M'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[df['SK_ID_CURR'] == 144669]['CODE_GENDER']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.loc[df.SK_ID_CURR == 141289, 'CODE_GENDER'] = 'F'\n",
    "df.loc[df.SK_ID_CURR == 319880, 'CODE_GENDER'] = 'F'\n",
    "df.loc[df.SK_ID_CURR == 196708, 'CODE_GENDER'] = 'F'\n",
    "df.loc[df.SK_ID_CURR == 144669, 'CODE_GENDER'] = 'M'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['ORGANIZATION_TYPE'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['OrganizationTypeGroup'] = ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.loc[df.ORGANIZATION_TYPE == 'Advertising', 'OrganizationTypeGroup'] = 'Advertising'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['OrganizationTypeGroup'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.drop('OrganizationTypeGroup', axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dict_organization_type_groups = {\n",
    "'Advertising':'Advertising',\n",
    "'Agriculture':'Agriculture',\n",
    "'Bank':'Bank',\n",
    "'Business_Entity':'Business Entity Type 1',\n",
    "'Business_Entity':'Business Entity Type 2',\n",
    "'Business_Entity':'Business Entity Type 3',\n",
    "'Cleaning':'Cleaning',\n",
    "'Construction':'Construction',\n",
    "'Culture':'Culture',\n",
    "'Electriciy':'Electricity',\n",
    "'Emergency':'Emergency',\n",
    "'Government':'Government',\n",
    "'Hotel':'Hotel',\n",
    "'Housing':'Housing',\n",
    "'Industry':'Industry: type 1',\n",
    "'Industry':'Industry: type 2',\n",
    "'Industry':'Industry: type 3',\n",
    "'Industry':'Industry: type 4',\n",
    "'Industry':'Industry: type 5',\n",
    "'Industry':'Industry: type 6',\n",
    "'Industry':'Industry: type 7',\n",
    "'Industry':'Industry: type 8',\n",
    "'Industry':'Industry: type 9',\n",
    "'Industry':'Industry: type 10',\n",
    "'Industry':'Industry: type 11',\n",
    "'Industry':'Industry: type 12',\n",
    "'Industry':'Industry: type 13',\n",
    "'Insurance':'Insurance',\n",
    "'Kindergarten':'Kindergarten',\n",
    "'Legal_Services':'Legal Services',\n",
    "'Medicine':'Medicine',\n",
    "'Military':'Military',\n",
    "'Mobile':'Mobile',\n",
    "'Other':'Other',\n",
    "'Police':'Police',\n",
    "'Postal':'Postal',\n",
    "'Realtor':'Realtor',\n",
    "'Religion':'Religion',\n",
    "'Restaurant':'Restaurant',\n",
    "'School':'School',\n",
    "'Security':'Security',\n",
    "'Security':'Security Ministries',\n",
    "'Self_Employed':'Self-employed',\n",
    "'Services':'Services',\n",
    "'Telecom':'Telecom',\n",
    "'Trade':'Trade: type 1',\n",
    "'Trade':'Trade: type 2',\n",
    "'Trade':'Trade: type 3',\n",
    "'Trade':'Trade: type 4',\n",
    "'Trade':'Trade: type 5',\n",
    "'Trade':'Trade: type 6',\n",
    "'Trade':'Trade: type 7',\n",
    "'Transport':'Transport: type 1',\n",
    "'Transport':'Transport: type 2',\n",
    "'Transport':'Transport: type 3',\n",
    "'Transport':'Transport: type 4',\n",
    "'University':'University',\n",
    "'XNA':'XNA'\n",
    "}\n",
    "dict_organization_type_groups"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_organization_type_groups = pd.DataFrame(dict_organization_type_groups)\n",
    "#> ValueError: If using all scalar values, you must pass an index\n",
    "#df_organization_type_groups = pd.Series(dict_organization_type_groups, name='OrganizationType')\n",
    "#df_organization_type_groups.index.name = 'OrganizationTypeGroup'\n",
    "#df_organization_type_groups.reset_index()\n",
    "#df_organization_type_groups\n",
    "df_organization_type_groups = pd.DataFrame(list(dict_organization_type_groups.items()), columns=['OrganizationTypeGroup', 'ORGANIZATION_TYPE'])  # Python 3\n",
    "df_organization_type_groups"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#application_train['OrganizationTypeGroup'] = application_train['OrganizationTypeGroup'].applymap(organization_type_groups.get)\n",
    "# pd.merge(left, right, how='inner', on=None, left_on=None, right_on=None,\n",
    "#         left_index=False, right_index=False, sort=True,\n",
    "#         suffixes=('_x', '_y'), copy=True, indicator=False,\n",
    "#         validate=None)\n",
    "df = pd.merge(application_train,\n",
    "                             df_organization_type_groups,\n",
    "                             how='right',\n",
    "                             left_on='ORGANIZATION_TYPE',\n",
    "                             right_on='ORGANIZATION_TYPE')\n",
    "df['OrganizationTypeGroup'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# FLAG_MOBIL, FLAG_EMP_PHONE, FLAG_WORK_PHONE, FLAG_CONT_MOBILE, FLAG_PHONE, FLAG_EMAIL  \n",
    "FLAG_MOBIL, FLAG_EMP_PHONE, FLAG_WORK_PHONE, FLAG_CONT_MOBILE, FLAG_PHONE, FLAG_EMAIL  \n",
    "FLAG_MOBIL        Did client provide mobile phone (1=YES, 0=NO)  \n",
    "FLAG_EMP_PHONE    Did client provide work phone (1=YES, 0=NO)  \n",
    "FLAG_WORK_PHONE   Did client provide home phone (1=YES, 0=NO)  \n",
    "FLAG_CONT_MOBILE  Was mobile phone reachable (1=YES, 0=NO)  \n",
    "FLAG_PHONE        Did client provide home phone (1=YES, 0=NO)  \n",
    "FLAG_EMAIL        Did client provide email (1=YES, 0=NO)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['FLAG_MOBIL'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['FLAG_EMP_PHONE'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['FLAG_WORK_PHONE'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['FLAG_CONT_MOBILE'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['FLAG_PHONE'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['FLAG_EMAIL'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DAYS_LAST_PHONE_CHANGE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['DAYS_LAST_PHONE_CHANGE'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# FLAG_OWN_REALTY, FLAG_OWN_CAR, OWN_CAR_AGE, NAME_HOUSING_TYPE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['FLAG_OWN_REALTY'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['FLAG_OWN_CAR'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['OWN_CAR_AGE'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['NAME_HOUSING_TYPE'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['NAME_CONTRACT_TYPE'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9047871458256778"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Cash loans\n",
    "278232 / df_row_count\n",
    "#> 0.9047871458256778"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.09521285417432222"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Revolving loans\n",
    "29279 / df_row_count\n",
    "#> 0.09521285417432222"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
